#100 support filesystem as a source

Author: arttor

README.md

@@ -19,16 +19,35 @@ Submit issue if some features missing for your use-case.
 
 ## Usage
 
-1) From file: `cat my-app.yaml | helmify mychart`
-    
-    Will create 'mychart' directory with Helm chart from yaml file with k8s objects.
+1) As pipe:
 
+    ```shell
+    cat my-app.yaml | helmify mychart
+    ```
+   Will create 'mychart' directory with Helm chart from yaml file with k8s objects.
 
-2) From directory with yamls:
     ```shell
     awk 'FNR==1 && NR!=1  {print "---"}{print}' /<my_directory>/*.yaml | helmify mychart
     ```
+   Will create 'mychart' directory with Helm chart from all yaml files in `<my_directory> `directory.
+
+2) From filesystem:
+    ```shell
+    helmify -f /my_directory/my-app.yaml mychart
+    ```
+    Will create 'mychart' directory with Helm chart from `my_directory/my-app.yaml`.
+    ```shell
+    helmify -f /my_directory mychart
+    ```
     Will create 'mychart' directory with Helm chart from all yaml files in `<my_directory> `directory.
+    ```shell
+    helmify -f /my_directory -r mychart
+    ```
+    Will create 'mychart' directory with Helm chart from all yaml files in `<my_directory> `directory recursively.
+    ```shell
+    helmify -f ./first_dir -f ./second_dir/my_deployment.yaml -f ./third_dir  mychart
+    ```
+    Will create 'mychart' directory with Helm chart from multiple directories and files.
 
 
 3) From [kustomize](https://kustomize.io/) output:
@@ -78,15 +97,17 @@ Usage:
 
 ```helmify [flags] CHART_NAME```  -  `CHART_NAME` is optional. Default is 'chart'. Can be a directory, e.g. 'deploy/charts/mychart'.
 
-| flag | description                                                                                                                                                                                                 | sample |
-| --- | --- | --- |
-| -h -help | Prints help                                                                                                                                                                                                 | `helmify -h`|
-| -v | Enable verbose output. Prints WARN and INFO.                                                                                                                                                                | `helmify -v`|
-| -vv | Enable very verbose output. Also prints DEBUG.                                                                                                                                                              | `helmify -vv`|
-| -version | Print helmify version.                                                                                                                                                                                      | `helmify -version`|
-| -crd-dir | Place crds in their own folder per Helm 3 [docs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you). Caveat: CRDs templating is not supported by Helm. | `helmify -crd-dir`|
-| -image-pull-secrets| Allows the user to use existing secrets as imagePullSecrets  | `helmify -image-pull-secrets`|
-| -cert-manager-as-subchart | Allows the user to install cert-manager as a subchart  | `helmify -cert-manager-as-subchart`|
+| flag                      | description                                                                                                                                                                                                 | sample                              |
+|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------|
+| -h -help                  | Prints help                                                                                                                                                                                                 | `helmify -h`                        |
+| -f                        | File source for k8s manifests (directory or file), multiple sources supported                                                                                                                               | `helmify -f ./test_data`            |
+| -r                        | Scan file directory recursively. Used only if -f provided                                                                                                                                                   | `helmify -f ./test_data -r`         |
+| -v                        | Enable verbose output. Prints WARN and INFO.                                                                                                                                                                | `helmify -v`                        |
+| -vv                       | Enable very verbose output. Also prints DEBUG.                                                                                                                                                              | `helmify -vv`                       |
+| -version                  | Print helmify version.                                                                                                                                                                                      | `helmify -version`                  |
+| -crd-dir                  | Place crds in their own folder per Helm 3 [docs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you). Caveat: CRDs templating is not supported by Helm. | `helmify -crd-dir`                  |
+| -image-pull-secrets       | Allows the user to use existing secrets as imagePullSecrets                                                                                                                                                 | `helmify -image-pull-secrets`       |
+| -cert-manager-as-subchart | Allows the user to install cert-manager as a subchart                                                                                                                                                       | `helmify -cert-manager-as-subchart` |
 ## Status
 Supported k8s resources:
 - Deployment, DaemonSet, StatefulSet

cmd/helmify/flags.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/arttor/helmify/pkg/config"
 )
@@ -17,7 +18,16 @@ Example 1: 'kustomize build <kustomize_dir> | helmify mychart'
 Example 2: 'cat my-app.yaml | helmify mychart' 
   - will create 'mychart' directory with Helm chart from yaml file.
 
-Example 3: 'awk 'FNR==1 && NR!=1  {print "---"}{print}' /my_directory/*.yaml | helmify mychart' 
+Example 3: 'helmify -f ./test_data/dir  mychart' 
+  - will scan directory ./test_data/dir for files with k8s manifests and create 'mychart' directory with Helm chart.
+
+Example 4: 'helmify -f ./test_data/dir -r  mychart' 
+  - will scan directory ./test_data/dir recursively and  create 'mychart' directory with Helm chart.
+
+Example 5: 'helmify -f ./test_data/dir -f ./test_data/sample-app.yaml -f ./test_data/dir/another_dir  mychart' 
+  - will scan provided multiple files and directories and  create 'mychart' directory with Helm chart.
+
+Example 6: 'awk 'FNR==1 && NR!=1  {print "---"}{print}' /my_directory/*.yaml | helmify mychart' 
   - will create 'mychart' directory with Helm chart from all yaml files in my_directory directory.
 
 Usage:
@@ -26,8 +36,23 @@ Usage:
 Flags:
 `
 
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	if i == nil || len(*i) == 0 {
+		return ""
+	}
+	return strings.Join(*i, ", ")
+}
+
+func (i *arrayFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
 // ReadFlags command-line flags into app config.
 func ReadFlags() config.Config {
+	files := arrayFlags{}
 	result := config.Config{}
 	var h, help, version, crd bool
 	flag.BoolVar(&h, "h", false, "Print help. Example: helmify -h")
@@ -39,6 +64,8 @@ func ReadFlags() config.Config {
 	flag.BoolVar(&result.ImagePullSecrets, "image-pull-secrets", false, "Allows the user to use existing secrets as imagePullSecrets in values.yaml")
 	flag.BoolVar(&result.GenerateDefaults, "generate-defaults", false, "Allows the user to add empty placeholders for tipical customization options in values.yaml. Currently covers: topology constraints, node selectors, tolerances")
 	flag.BoolVar(&result.CertManagerAsSubchart, "cert-manager-as-subchart", false, "Allows the user to add cert-manager as a subchart")
+	flag.BoolVar(&result.FilesRecursively, "r", false, "Scan dirs from -f option recursively")
+	flag.Var(&files, "f", "File or directory containing k8s manifests")
 
 	flag.Parse()
 	if h || help {
@@ -58,5 +85,6 @@ func ReadFlags() config.Config {
 	if crd {
 		result.Crd = crd
 	}
+	result.Files = files
 	return result
 }

cmd/helmify/main.go

@@ -14,7 +14,7 @@ func main() {
 		logrus.WithError(err).Error("stdin error")
 		os.Exit(1)
 	}
-	if (stat.Mode() & os.ModeCharDevice) != 0 {
+	if len(conf.Files) == 0 && (stat.Mode()&os.ModeCharDevice) != 0 {
 		logrus.Error("no data piped in stdin")
 		os.Exit(1)
 	}

pkg/app/app.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"github.com/arttor/helmify/pkg/file"
 	"github.com/arttor/helmify/pkg/processor/job"
 	"github.com/arttor/helmify/pkg/processor/statefulset"
 	"io"
@@ -27,7 +28,7 @@ import (
 )
 
 // Start - application entrypoint for processing input to a Helm chart.
-func Start(input io.Reader, config config.Config) error {
+func Start(stdin io.Reader, config config.Config) error {
 	err := config.Validate()
 	if err != nil {
 		return err
@@ -42,7 +43,6 @@ func Start(input io.Reader, config config.Config) error {
 		logrus.Debug("Received termination, signaling shutdown")
 		cancelFunc()
 	}()
-	objects := decoder.Decode(ctx.Done(), input)
 	appCtx := New(config, helm.NewOutput())
 	appCtx = appCtx.WithProcessors(
 		configmap.New(),
@@ -65,9 +65,20 @@ func Start(input io.Reader, config config.Config) error {
 		job.NewCron(),
 		job.NewJob(),
 	).WithDefaultProcessor(processor.Default())
-	for obj := range objects {
-		appCtx.Add(obj)
+	if len(config.Files) != 0 {
+		file.Walk(config.Files, config.FilesRecursively, func(filename string, fileReader io.Reader) {
+			objects := decoder.Decode(ctx.Done(), fileReader)
+			for obj := range objects {
+				appCtx.Add(obj, filename)
+			}
+		})
+	} else {
+		objects := decoder.Decode(ctx.Done(), stdin)
+		for obj := range objects {
+			appCtx.Add(obj, "")
+		}
 	}
+
 	return appCtx.CreateHelm(ctx.Done())
 }
 

pkg/app/context.go

@@ -16,6 +16,7 @@ type appContext struct {
 	config           config.Config
 	appMeta          *metadata.Service
 	objects          []*unstructured.Unstructured
+	fileNames        []string
 }
 
 // New returns context with config set.
@@ -40,10 +41,11 @@ func (c *appContext) WithDefaultProcessor(processor helmify.Processor) *appConte
 }
 
 // Add k8s object to app context.
-func (c *appContext) Add(obj *unstructured.Unstructured) {
+func (c *appContext) Add(obj *unstructured.Unstructured, filename string) {
 	// we need to add all objects before start processing only to define app metadata.
 	c.appMeta.Load(obj)
 	c.objects = append(c.objects, obj)
+	c.fileNames = append(c.fileNames, filename)
 }
 
 // CreateHelm creates helm chart from context k8s objects.
@@ -53,21 +55,27 @@ func (c *appContext) CreateHelm(stop <-chan struct{}) error {
 		"Namespace": c.appMeta.Namespace(),
 	}).Info("creating a chart")
 	var templates []helmify.Template
-	for _, obj := range c.objects {
+	var filenames []string
+	for i, obj := range c.objects {
 		template, err := c.process(obj)
 		if err != nil {
 			return err
 		}
 		if template != nil {
 			templates = append(templates, template)
+			filename := template.Filename()
+			if c.fileNames[i] != "" {
+				filename = c.fileNames[i]
+			}
+			filenames = append(filenames, filename)
 		}
 		select {
 		case <-stop:
 			return nil
 		default:
 		}
 	}
-	return c.output.Create(c.config.ChartDir, c.config.ChartName, c.config.Crd, c.config.CertManagerAsSubchart, templates)
+	return c.output.Create(c.config.ChartDir, c.config.ChartName, c.config.Crd, c.config.CertManagerAsSubchart, templates, filenames)
 }
 
 func (c *appContext) process(obj *unstructured.Unstructured) (helmify.Template, error) {

pkg/config/config.go

@@ -28,6 +28,10 @@ type Config struct {
 	GenerateDefaults bool
 	// CertManagerAsSubchart enables the generation of a subchart for cert-manager
 	CertManagerAsSubchart bool
+	// Files - directories or files with k8s manifests
+	Files []string
+	// FilesRecursively read Files recursively
+	FilesRecursively bool
 }
 
 func (c *Config) Validate() error {

pkg/file/reader.go

@@ -0,0 +1,87 @@
+package file
+
+import (
+	"github.com/sirupsen/logrus"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+func Walk(paths []string, recursively bool, walkFunc func(filename string, r io.Reader)) {
+
+	for _, path := range paths {
+		info, err := os.Stat(path)
+		if err != nil {
+			logrus.Warnf("no such file or directory %q: %v", path, err)
+			continue
+		}
+		// handle single file file:
+		if !info.IsDir() {
+			file, err := os.Open(path)
+			if err != nil {
+				logrus.Warnf("unable to open file %q: %v", file.Name(), err)
+				continue
+			}
+			walkFunc(info.Name(), file)
+			err = file.Close()
+			if err != nil {
+				logrus.Warnf("unable to close file %q: %v", file.Name(), err)
+			}
+			continue
+		}
+		// handle directory non-recursively:
+		if !recursively {
+			dir, err := os.Open(path)
+			if err != nil {
+				logrus.Warnf("unable to open directory %q: %v", dir.Name(), err)
+				continue
+			}
+			files, err := dir.ReadDir(0)
+			if err != nil {
+				logrus.Warnf("unable to read directory %q: %v", dir.Name(), err)
+				continue
+			}
+			for _, f := range files {
+				if f.IsDir() {
+					continue
+				}
+				file, err := os.Open(filepath.Join(path, f.Name()))
+				if err != nil {
+					logrus.Warnf("unable to open file %q: %v", file.Name(), err)
+					continue
+				}
+				walkFunc(f.Name(), file)
+				err = file.Close()
+				if err != nil {
+					logrus.Warnf("unable to close file %q: %v", file.Name(), err)
+				}
+				continue
+			}
+			continue
+		}
+		// handle directory recursively:
+		err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			walkFunc(d.Name(), file)
+			err = file.Close()
+			if err != nil {
+				logrus.Warnf("unable to close file %q: %v", file.Name(), err)
+			}
+			return nil
+		})
+		if err != nil {
+			logrus.Warnf("unable to open %q: %v", info.Name(), err)
+			continue
+		}
+	}
+}

pkg/helm/chart.go

@@ -31,7 +31,7 @@ type output struct{}
 //	    └── _helpers.tp   # Helm default template partials
 //
 // Overwrites existing values.yaml and templates in templates dir on every run.
-func (o output) Create(chartDir, chartName string, crd bool, certManagerAsSubchart bool, templates []helmify.Template) error {
+func (o output) Create(chartDir, chartName string, crd bool, certManagerAsSubchart bool, templates []helmify.Template, filenames []string) error {
 	err := initChartDir(chartDir, chartName, crd, certManagerAsSubchart)
 	if err != nil {
 		return err
@@ -40,10 +40,10 @@ func (o output) Create(chartDir, chartName string, crd bool, certManagerAsSubcha
 	files := map[string][]helmify.Template{}
 	values := helmify.Values{}
 	values[cluster.DomainKey] = cluster.DefaultDomain
-	for _, template := range templates {
-		file := files[template.Filename()]
+	for i, template := range templates {
+		file := files[filenames[i]]
 		file = append(file, template)
-		files[template.Filename()] = file
+		files[filenames[i]] = file
 		err = values.Merge(template.Values())
 		if err != nil {
 			return err

pkg/helmify/model.go

@@ -28,7 +28,7 @@ type Template interface {
 
 // Output - converts Template into helm chart on disk.
 type Output interface {
-	Create(chartName, chartDir string, Crd bool, certManagerAsSubchart bool, templates []Template) error
+	Create(chartName, chartDir string, Crd bool, certManagerAsSubchart bool, templates []Template, filenames []string) error
 }
 
 // AppMetadata handle common information about K8s objects in the chart.