#30 multiline configmap values

Author: arttor

examples/app/templates/my-config-props.yaml

@@ -7,4 +7,5 @@ metadata:
 data:
   my.prop1: {{ .Values.myConfigProps.myProp1 | quote }}
   my.prop2: {{ .Values.myConfigProps.myProp2 | quote }}
-  my.prop3: {{ .Values.myConfigProps.myProp3 | quote }}
+  my.prop3: {{ .Values.myConfigProps.myProp3 | quote }}
+  myval.yaml: {{ .Values.myConfigProps.myvalYaml | toYaml | indent 1 }}

examples/app/values.yaml

@@ -21,6 +21,16 @@ myConfigProps:
   myProp1: "1"
   myProp2: val 1
   myProp3: "true"
+  myvalYaml: |-
+    apiVersion: clickhouse.altinity.com/v1
+    kind: ClickHouseInstallationTemplate
+    metadata:
+      name: default-oneperhost-pod-template
+    spec:
+      templates:
+        podTemplates:
+          - name: default-oneperhost-pod-template
+            distribution: "OnePerHost"
 mySecretCa:
   caCrt: ""
 mySecretVars:

examples/operator/templates/manager-config.yaml

@@ -5,25 +5,6 @@ metadata:
   labels:
   {{- include "operator.labels" . | nindent 4 }}
 data:
-  controller_manager_config.yaml: |
-    apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
-    health:
-      healthProbeBindAddress: {{ .Values.managerConfig.controllerManagerConfigYaml.health.healthProbeBindAddress
-        | quote }}
-    kind: ControllerManagerConfig
-    leaderElection:
-      leaderElect: {{ .Values.managerConfig.controllerManagerConfigYaml.leaderElection.leaderElect
-        }}
-      resourceName: {{ .Values.managerConfig.controllerManagerConfigYaml.leaderElection.resourceName
-        | quote }}
-    metrics:
-      bindAddress: {{ .Values.managerConfig.controllerManagerConfigYaml.metrics.bindAddress
-        | quote }}
-    rook:
-      namespace: {{ .Values.managerConfig.controllerManagerConfigYaml.rook.namespace
-        | quote }}
-      toolboxPodLabel: {{ .Values.managerConfig.controllerManagerConfigYaml.rook.toolboxPodLabel
-        | quote }}
-    webhook:
-      port: {{ .Values.managerConfig.controllerManagerConfigYaml.webhook.port }}
+  controller_manager_config.yaml: {{ .Values.managerConfig.controllerManagerConfigYaml
+    | toYaml | indent 1 }}
   dummyconfigmapkey: {{ .Values.managerConfig.dummyconfigmapkey | quote }}

examples/operator/values.yaml

@@ -37,19 +37,21 @@ controllerManager:
   replicas: 1
 kubernetesClusterDomain: cluster.local
 managerConfig:
-  controllerManagerConfigYaml:
+  controllerManagerConfigYaml: |-
+    apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
+    kind: ControllerManagerConfig
     health:
       healthProbeBindAddress: :8081
+    metrics:
+      bindAddress: 127.0.0.1:8080
+    webhook:
+      port: 9443
     leaderElection:
       leaderElect: true
       resourceName: 3a2e09e9.example.com
-    metrics:
-      bindAddress: 127.0.0.1:8080
     rook:
       namespace: rook-ceph
       toolboxPodLabel: rook-ceph-tools
-    webhook:
-      port: 9443
   dummyconfigmapkey: dummyconfigmapvalue
 metricsService:
   ports:

pkg/format/trailing_whitespaces.go

@@ -0,0 +1,9 @@
+package format
+
+import "regexp"
+
+var removeWhitespace = regexp.MustCompile(`(\s+)(\n|$)`)
+
+func RemoveTrailingWhitespaces(in string) string {
+	return removeWhitespace.ReplaceAllString(in, "$2")
+}

pkg/format/trailing_whitespaces_test.go

@@ -0,0 +1,45 @@
+package format
+
+import "testing"
+
+func TestRemoveTrailingWhitespaces(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{
+			name: "",
+			in:   `abc   `,
+			want: `abc`,
+		},
+		{
+			name: "",
+			in: `abc   
+edf`,
+			want: `abc
+edf`,
+		},
+		{
+			name: "",
+			in: `abc   
+edf   `,
+			want: `abc
+edf`,
+		},
+		{
+			name: "",
+			in: `abc   .
+edf   .`,
+			want: `abc   .
+edf   .`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RemoveTrailingWhitespaces(tt.in); got != tt.want {
+				t.Errorf("RemoveTrailingWhitespaces() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/helmify/values.go

@@ -41,6 +41,20 @@ func (v *Values) Add(value interface{}, name ...string) (string, error) {
 	return "{{ .Values." + strings.Join(name, ".") + " }}", nil
 }
 
+// AddYaml - adds given value to values and returns its helm template representation as Yaml {{ .Values.<valueName> | toYaml | indent i }}
+// indent  <= 0 will be omitted.
+func (v *Values) AddYaml(value interface{}, indent int, name ...string) (string, error) {
+	name = toCamelCase(name)
+	err := unstructured.SetNestedField(*v, value, name...)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to set value: %v", name)
+	}
+	if indent > 0 {
+		return "{{ .Values." + strings.Join(name, ".") + fmt.Sprintf(" | toYaml | indent %d }}", indent), nil
+	}
+	return "{{ .Values." + strings.Join(name, ".") + " | toYaml }}", nil
+}
+
 // AddSecret - adds empty value to values and returns its helm template representation {{ required "<valueName>" .Values.<valueName> }}.
 // Set toBase64=true for Secret data to be base64 encoded and set false for Secret stringData.
 func (v *Values) AddSecret(toBase64 bool, name ...string) (string, error) {

pkg/processor/configmap/configmap.go

@@ -1,7 +1,7 @@
 package configmap
 
 import (
-	"fmt"
+	"github.com/arttor/helmify/pkg/format"
 	"io"
 	"strings"
 	"text/template"
@@ -14,7 +14,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/yaml"
 )
 
 var configMapTempl, _ = template.New("configMap").Parse(
@@ -68,7 +67,7 @@ func (d configMap) Process(appMeta helmify.AppMetadata, obj *unstructured.Unstru
 
 	name := appMeta.TrimName(obj.GetName())
 	var values helmify.Values
-	if field, exists, _ := unstructured.NestedMap(obj.Object, "data"); exists {
+	if field, exists, _ := unstructured.NestedStringMap(obj.Object, "data"); exists {
 		field, values = parseMapData(field, name)
 		data, err = yamlformat.Marshal(map[string]interface{}{"data": field}, 0)
 		if err != nil {
@@ -89,28 +88,31 @@ func (d configMap) Process(appMeta helmify.AppMetadata, obj *unstructured.Unstru
 	}, nil
 }
 
-func parseMapData(data map[string]interface{}, configName string) (map[string]interface{}, helmify.Values) {
+func parseMapData(data map[string]string, configName string) (map[string]string, helmify.Values) {
 	values := helmify.Values{}
 	for key, value := range data {
 		valuesNamePath := []string{configName, key}
-		if strings.HasSuffix(key, ".yaml") || strings.HasSuffix(key, ".yml") {
-			templated, err := parseYaml(value, valuesNamePath, values)
+		if strings.HasSuffix(key, ".properties") {
+			// handle properties
+			templated, err := parseProperties(value, valuesNamePath, values)
 			if err != nil {
 				logrus.WithError(err).Errorf("unable to process configmap data: %v", valuesNamePath)
 				continue
 			}
 			data[key] = templated
 			continue
 		}
-		if strings.HasSuffix(key, ".properties") {
-			templated, err := parseProperties(value, valuesNamePath, values)
+		if strings.Contains(value, "\n") {
+			value = format.RemoveTrailingWhitespaces(value)
+			templatedVal, err := values.AddYaml(value, 1, valuesNamePath...)
 			if err != nil {
-				logrus.WithError(err).Errorf("unable to process configmap data: %v", valuesNamePath)
+				logrus.WithError(err).Errorf("unable to process multiline configmap data: %v", valuesNamePath)
 				continue
 			}
-			data[key] = templated
+			data[key] = templatedVal
 			continue
 		}
+		// handle plain string
 		templatedVal, err := values.Add(value, valuesNamePath...)
 		if err != nil {
 			logrus.WithError(err).Errorf("unable to process configmap data: %v", valuesNamePath)
@@ -121,20 +123,6 @@ func parseMapData(data map[string]interface{}, configName string) (map[string]in
 	return data, values
 }
 
-func parseYaml(value interface{}, path []string, values helmify.Values) (string, error) {
-	config := map[string]interface{}{}
-	err := yaml.Unmarshal([]byte(value.(string)), &config)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to unmarshal configmap %v", path)
-	}
-	parseConfig(config, values, path)
-	confBytes, err := yaml.Marshal(config)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to marshal configmap %v", path)
-	}
-	return string(confBytes), nil
-}
-
 // func parseProperties(properties string, path []string, values helmify.Values) (string, error) {
 func parseProperties(properties interface{}, path []string, values helmify.Values) (string, error) {
 	var res strings.Builder
@@ -157,51 +145,6 @@ func parseProperties(properties interface{}, path []string, values helmify.Value
 	return res.String(), nil
 }
 
-func parseConfig(config map[string]interface{}, values helmify.Values, path []string) {
-	for k, v := range config {
-		switch t := v.(type) {
-		case string, bool, float64, int64:
-			if k == "kind" || k == "apiVersion" {
-				continue
-			}
-			templated, err := values.Add(v, append(path, k)...)
-			if err != nil {
-				logrus.WithError(err).Error()
-				continue
-			}
-			config[k] = templated
-		case []interface{}:
-			templated, err := values.Add(v, append(path, k)...)
-			if err != nil {
-				logrus.WithError(err).Error()
-				continue
-			}
-			config[k] = templated
-		case map[string]interface{}:
-			if len(t) == 0 {
-				templated, err := values.Add(v, append(path, k)...)
-				if err != nil {
-					logrus.WithError(err).Error()
-					continue
-				}
-				config[k] = templated
-				continue
-			}
-			parseConfig(t, values, append(path, k))
-		case map[interface{}]interface{}:
-			c, ok := v.(map[string]interface{})
-			if !ok {
-				logrus.Warn("configmap: unable to cast to map[string]interface{}")
-				continue
-			}
-			parseConfig(c, values, append(path, k))
-		default:
-			logrus.Warn("configmap: unknown type ", t)
-			fmt.Printf("\n%T\n", t)
-		}
-	}
-}
-
 type result struct {
 	name string
 	data struct {

test_data/sample-app.yaml

@@ -193,6 +193,16 @@ data:
   my.prop1: "1"
   my.prop2: "val 1"
   my.prop3: "true"
+  myval.yaml: |
+    apiVersion: clickhouse.altinity.com/v1
+    kind: ClickHouseInstallationTemplate
+    metadata:
+      name: default-oneperhost-pod-template
+    spec:
+      templates:
+        podTemplates: 
+          - name: default-oneperhost-pod-template
+            distribution: "OnePerHost"
 ---
 apiVersion: apps/v1
 kind: DaemonSet