feat: nested manifests in object transforms

This change adds support for applying object transforms to nested
manifests. The nested manifest pattern is a pattern in which an object
has a string field which consists of its own yaml manifest.

As an example, a ConfigMap may contain a manifest bundle in one of its
data fields.

A nested manifest will recurse and apply all object transforms to the
underlying objects and then write the result back to the parent object.

Author: sdowell

pkg/patterns/declarative/options.go

@@ -35,6 +35,9 @@ var DefaultManifestLoader ManifestLoaderFunc
 // ReconcilerOption implements the options pattern for reconcilers
 type ReconcilerOption func(params reconcilerParams) reconcilerParams
 
+// NestedManifestFunc returns the path to any nested manifests in the object, if any.
+type NestedManifestFunc func(m *manifest.Object) ([][]string, error)
+
 // Options are a set of reconcilerOptions applied to all controllers
 var Options struct {
 	// Begin options are applied before evaluating controller specific options
@@ -64,6 +67,12 @@ type reconcilerParams struct {
 
 	// hooks allow for interception of events during the reconciliation lifecycle
 	hooks []Hook
+
+	// nestedManifestFn allows specifying whether the object contains nested
+	// manifests. A nested manifest is a field which contains its own yaml bundle.
+	// The nested manifest will be unmarshalled, object transforms will be applied,
+	// and the result is then marshalled back to the parent object.
+	nestedManifestFn NestedManifestFunc
 }
 
 type ManifestController interface {
@@ -252,3 +261,16 @@ func WithHook(hook Hook) ReconcilerOption {
 		return p
 	}
 }
+
+// WithNestedManifestFunc allows specifying whether the object contains nested
+// manifests. A nested manifest is a field which contains its own yaml bundle.
+// The nested manifest will be unmarshalled, object transforms will be applied,
+// and the result is then marshalled back to the parent object.
+// The provided function should return a list of field paths where nested manifests
+// are expected to be found.
+func WithNestedManifestFunc(nestedManifestFn NestedManifestFunc) ReconcilerOption {
+	return func(p reconcilerParams) reconcilerParams {
+		p.nestedManifestFn = nestedManifestFn
+		return p
+	}
+}

pkg/patterns/declarative/pkg/manifest/objects.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/yaml"
 )
 
 // Objects holds a collection of objects, so that we can filter / sequence them
@@ -363,6 +364,23 @@ func (o *Objects) JSONManifest() (string, error) {
 	return b.String(), nil
 }
 
+// ToYAML marshals the list of objects to a yaml manifest
+func (o *Objects) ToYAML() (string, error) {
+	var b bytes.Buffer
+
+	for i, item := range o.Items {
+		objYaml, err := yaml.Marshal(item.UnstructuredObject().Object)
+		if err != nil {
+			return "", err
+		}
+		b.Write(objYaml)
+		if i < len(o.Items)-1 {
+			b.WriteString("---\n")
+		}
+	}
+	return b.String(), nil
+}
+
 // Sort will order the items in Objects in order of score, group, kind, name.  The intent is to
 // have a deterministic ordering in which Objects are applied.
 func (o *Objects) Sort(score func(o *Object) int) {

pkg/patterns/declarative/pkg/manifest/objects_test.go

@@ -1269,3 +1269,135 @@ func Test_Sort(t *testing.T) {
 		})
 	}
 }
+
+func Test_ToYAML(t *testing.T) {
+	deployment := &Object{
+		object: &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"apiVersion": "apps/v1",
+				"kind":       "Deployment",
+				"metadata": map[string]interface{}{
+					"name": "frontend111",
+				},
+				"spec": map[string]interface{}{
+					"template": map[string]interface{}{
+						"spec": map[string]interface{}{
+							"containers": []map[string]interface{}{
+								{
+									"name":  "php-redis",
+									"image": "gcr.io/google-samples/gb-frontend:v4",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		name:  "frontend111",
+		Kind:  "Deployment",
+		Group: "apps",
+	}
+	service := &Object{
+		object: &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name": "frontend-service",
+				},
+			},
+		},
+		name:  "frontend-service",
+		Kind:  "Service",
+		Group: "",
+	}
+	serviceAccount := &Object{
+		object: &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "ServiceAccount",
+				"metadata": map[string]interface{}{
+					"name": "serviceaccount",
+				},
+			},
+		},
+		name:  "serviceaccount",
+		Kind:  "ServiceAccount",
+		Group: "",
+	}
+	tests := []struct {
+		name           string
+		inputObjects   *Objects
+		expectedOutput string
+	}{
+		{
+			name: "multiple objects",
+			inputObjects: &Objects{
+				Items: []*Object{
+					deployment,
+					service,
+					serviceAccount,
+				},
+			},
+			expectedOutput: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend111
+spec:
+  template:
+    spec:
+      containers:
+      - image: gcr.io/google-samples/gb-frontend:v4
+        name: php-redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend-service
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: serviceaccount
+`,
+		},
+		{
+			name: "empty list",
+			inputObjects: &Objects{
+				Items: []*Object{},
+			},
+			expectedOutput: ``,
+		},
+		{
+			name: "single object",
+			inputObjects: &Objects{
+				Items: []*Object{
+					deployment,
+				},
+			},
+			expectedOutput: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend111
+spec:
+  template:
+    spec:
+      containers:
+      - image: gcr.io/google-samples/gb-frontend:v4
+        name: php-redis
+`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := tt.inputObjects.ToYAML()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(tt.expectedOutput, out); diff != "" {
+				t.Errorf("output yaml mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

pkg/patterns/declarative/reconciler.go

@@ -599,6 +599,47 @@ func (r *Reconciler) transformManifest(ctx context.Context, instance Declarative
 			return err
 		}
 	}
+	if r.options.nestedManifestFn != nil {
+		if err := r.transformNestedManifests(ctx, instance, objects); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *Reconciler) transformNestedManifests(ctx context.Context, instance DeclarativeObject, objects *manifest.Objects) error {
+	for _, item := range objects.Items {
+		paths, err := r.options.nestedManifestFn(item)
+		if err != nil {
+			return err
+		}
+		if paths == nil {
+			continue
+		}
+		for _, path := range paths {
+			nestedString, found, err := unstructured.NestedString(item.UnstructuredObject().Object, path...)
+			if err != nil {
+				return err
+			}
+			if !found { // Should this not be treated as an error?
+				return fmt.Errorf("expected object to have path %v", err)
+			}
+			nestedManifest, err := manifest.ParseObjects(ctx, nestedString)
+			if err != nil {
+				return err
+			}
+			if err := r.transformManifest(ctx, instance, nestedManifest); err != nil {
+				return err
+			}
+			updatedString, err := nestedManifest.ToYAML()
+			if err != nil {
+				return err
+			}
+			if err := unstructured.SetNestedField(item.UnstructuredObject().Object, updatedString, path...); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }