Update golden test framework

* Move to envtest by default (instead of mock-kubeapiserver)

* Create shared code for capturing requests / normalization.

* Use a more conventional env var WRITE_GOLDEN_OUTPUT

* Support object rewriting

Author: justinsb

dev/test

@@ -27,6 +27,9 @@ cd "${REPO_ROOT}"
 
 set -x
 
+# Download the kubebuilder assets for envtest
+export KUBEBUILDER_ASSETS=$(go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest use -p path)
+
 pushd mockkubeapiserver
 CGO_ENABLED=0 go test -count=1 -v ./...
 popd

docs/addon/walkthrough/tests.md

@@ -33,7 +33,7 @@ This means that the output is materialized and checked in to the repo; this
 proves to be very handy for understanding the impact of a change.
 
 There's also a helpful "cheat" function, which rewrite the output when you run
-the tests locally - set the HACK_AUTOFIX_EXPECTED_OUTPUT env var to a non-empty
+the tests locally - set the WRITE_GOLDEN_OUTPUT env var to a non-empty
 string.  This is useful when you have a big set of changes; it's just as easy to
 review the changes yourself in the diff and there's not a ton of value in typing
 them out.
@@ -87,7 +87,7 @@ the env-var cheat code.
    ```bash
    cd pkg/controller/{{operator}}/tests
    touch tests/simple.out.yaml
-   HACK_AUTOFIX_EXPECTED_OUTPUT=1 go test ./...
+   WRITE_GOLDEN_OUTPUT=1 go test ./...
    ```
 
 1. Verify the output is reproducible

examples/guestbook-operator/controllers/guestbook_controller_test.go

@@ -17,14 +17,28 @@ limitations under the License.
 package controllers
 
 import (
+	"path/filepath"
 	"testing"
 
+	"k8s.io/klog/v2/klogr"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	api "sigs.k8s.io/kubebuilder-declarative-pattern/examples/guestbook-operator/api/v1alpha1"
 	"sigs.k8s.io/kubebuilder-declarative-pattern/pkg/test/golden"
 )
 
 func TestGuestbook(t *testing.T) {
-	v := golden.NewValidator(t, api.SchemeBuilder)
+	log.SetLogger(klogr.New())
+
+	env := &envtest.Environment{
+		CRDInstallOptions: envtest.CRDInstallOptions{
+			ErrorIfPathMissing: true,
+			Paths: []string{
+				filepath.Join("..", "config", "crd", "bases"),
+			},
+		},
+	}
+	v := golden.NewValidator(t, env, api.AddToScheme)
 	dr := &GuestbookReconciler{
 		Client: v.Client(),
 	}
@@ -33,5 +47,5 @@ func TestGuestbook(t *testing.T) {
 		t.Fatalf("creating reconciler: %v", err)
 	}
 
-	v.Validate(dr.Reconciler)
+	v.Validate(&dr.Reconciler, golden.ValidateOptions{})
 }

examples/guestbook-operator/controllers/tests/patches-stable.in.yaml

@@ -2,6 +2,7 @@ apiVersion: addons.example.org/v1alpha1
 kind: Guestbook
 metadata:
   name: guestbook-sample
+  namespace: default
 spec:
   channel: stable
   patches:

examples/guestbook-operator/controllers/tests/simple-stable.in.yaml

@@ -2,5 +2,6 @@ apiVersion: addons.example.org/v1alpha1
 kind: Guestbook
 metadata:
   name: guestbook-sample
+  namespace: default
 spec:
   channel: stable

ktest/httprecorder/kube_normalize.go

@@ -0,0 +1,100 @@
+package httprecorder
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/kubebuilder-declarative-pattern/applylib/forked/k8s.io/apimachinery/pkg/util/sets"
+)
+
+// placeholderTime is a placeholder time that is used to replace the timestamps, so that we can golden test.
+var placeholderTime = metav1.Date(2025, 4, 1, 0, 0, 0, 0, time.UTC)
+
+// NormalizeKubeRequestLog normalizes a kube request log for golden testing,
+// it replaces ephemeral values with placeholder values,
+// sorts requests into a predictable order,
+// and removes non-deterministic headers.
+func NormalizeKubeRequestLog(t *testing.T, requestLog *RequestLog, restConfig *rest.Config) {
+	// Whether we're using a proxy or not (http vs https), IPv6 or not, we want to normalize the host to kube-apiserver
+	apiserverRealHost := strings.ReplaceAll(restConfig.Host, "[::]", "127.0.0.1")
+	apiserverRealHost = strings.TrimPrefix(apiserverRealHost, "https://")
+	apiserverRealHost = strings.TrimPrefix(apiserverRealHost, "http://")
+	apiserverRealHost = strings.TrimSuffix(apiserverRealHost, "/")
+	requestLog.ReplaceURLPrefix("https://"+apiserverRealHost, "https://kube-apiserver")
+	requestLog.ReplaceURLPrefix("http://"+apiserverRealHost, "https://kube-apiserver")
+	apiserverRealHost = strings.ReplaceAll(apiserverRealHost, "127.0.0.1", "[::]")
+	requestLog.ReplaceURLPrefix("https://"+apiserverRealHost, "https://kube-apiserver")
+	requestLog.ReplaceURLPrefix("http://"+apiserverRealHost, "https://kube-apiserver")
+
+	requestLog.RemoveUserAgent()
+
+	requestLog.RemoveHeader("X-Kubernetes-Pf-Flowschema-Uid")
+	requestLog.RemoveHeader("X-Kubernetes-Pf-Prioritylevel-Uid")
+	requestLog.RemoveHeader("Audit-Id")
+
+	requestLog.RewriteBodies(t, func(body map[string]any) {
+		u := unstructured.Unstructured{Object: body}
+		uid := u.GetUID()
+		if uid != "" {
+			u.SetUID("fake-uid")
+		}
+
+		replaceIfPresent(t, body, placeholderTime.Format(time.RFC3339), "metadata", "creationTimestamp")
+
+		if managedFields := u.GetManagedFields(); managedFields != nil {
+			for i := range managedFields {
+				managedFields[i].Time = &placeholderTime
+			}
+			u.SetManagedFields(managedFields)
+		}
+	})
+
+	// Rewrite the resource version to a predictable value
+	{
+		resourceVersions := sets.New[int]()
+		requestLog.RewriteBodies(t, func(body map[string]any) {
+			u := unstructured.Unstructured{Object: body}
+			rv := u.GetResourceVersion()
+			if rv != "" {
+				// These aren't guaranteed to be ints, but in practice they are, and this is test code.
+				rvInt, err := strconv.Atoi(rv)
+				if err != nil {
+					t.Errorf("error converting resource version %q to int: %v", rv, err)
+					return
+				}
+				resourceVersions.Insert(rvInt)
+			}
+		})
+		resourceVersionsList := resourceVersions.UnsortedList()
+		sort.Ints(resourceVersionsList)
+		rewriteResourceVersion := map[string]string{}
+		for i, rv := range resourceVersionsList {
+			rewriteResourceVersion[strconv.Itoa(rv)] = strconv.Itoa(1000 + i)
+		}
+		requestLog.RewriteBodies(t, func(body map[string]any) {
+			u := unstructured.Unstructured{Object: body}
+			rv := u.GetResourceVersion()
+			u.SetResourceVersion(rewriteResourceVersion[rv])
+		})
+	}
+
+	requestLog.SortGETs()
+}
+
+func replaceIfPresent(t *testing.T, body map[string]any, replace string, path ...string) {
+	u := unstructured.Unstructured{Object: body}
+	_, found, err := unstructured.NestedFieldNoCopy(u.Object, path...)
+	if err != nil {
+		t.Errorf("error getting nested field %v: %v", path, err)
+		return
+	}
+	if found {
+		unstructured.SetNestedField(u.Object, replace, path...)
+	}
+}

ktest/httprecorder/request_log.go

@@ -70,7 +70,7 @@ func writeBody(w io.StringWriter, body string, pretty bool) {
 	}
 
 	if pretty {
-		var obj interface{}
+		var obj any
 		if err := json.Unmarshal([]byte(body), &obj); err == nil {
 			b, err := json.MarshalIndent(obj, "", "  ")
 			if err == nil {
@@ -186,8 +186,11 @@ func (l *RequestLog) RemoveHeader(k string) {
 	defer l.mutex.Unlock()
 
 	for i := range l.entries {
-		r := &l.entries[i].Request
-		r.Header.Del(k)
+		request := &l.entries[i].Request
+		request.Header.Del(k)
+
+		response := &l.entries[i].Response
+		response.Header.Del(k)
 	}
 }
 
@@ -260,3 +263,44 @@ func (l *RequestLog) RegexReplaceURL(t *testing.T, find string, replace string)
 		request.URL = u
 	}
 }
+
+// RewriteBodies rewrites the bodies of the requests and responses.
+// The function fn is called with the body as a map[string]any.
+func (l *RequestLog) RewriteBodies(t *testing.T, fn func(body map[string]any)) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	for i := range l.entries {
+		request := &l.entries[i].Request
+		request.Body = rewriteBody(t, request.Body, fn)
+
+		response := &l.entries[i].Response
+		response.Body = rewriteBody(t, response.Body, fn)
+	}
+}
+
+// rewriteBody rewrites the body of a request or response, assuming it is a JSON object.
+func rewriteBody(t *testing.T, body string, fn func(body map[string]any)) string {
+	if body == "" {
+		return body
+	}
+
+	var obj any
+	if err := json.Unmarshal([]byte(body), &obj); err != nil {
+		t.Errorf("failed to unmarshal body: %v", err)
+		return body
+	}
+
+	m, ok := obj.(map[string]any)
+	if !ok {
+		return body
+	}
+	fn(m)
+
+	b, err := json.MarshalIndent(obj, "", "  ")
+	if err != nil {
+		t.Errorf("failed to marshal body: %v", err)
+		return body
+	}
+	return string(b)
+}

ktest/testharness/golden.go

@@ -39,8 +39,12 @@ func RunGoldenTests(t *testing.T, basedir string, fn func(h *Harness, dir string
 	}
 }
 
+func ShouldWriteGoldenOutput() bool {
+	return os.Getenv("WRITE_GOLDEN_OUTPUT") != ""
+}
+
 func (h *Harness) CompareGoldenFile(p string, got string) {
-	if os.Getenv("WRITE_GOLDEN_OUTPUT") != "" {
+	if ShouldWriteGoldenOutput() {
 		// Short-circuit when the output is correct
 		b, err := os.ReadFile(p)
 		if err == nil && bytes.Equal(b, []byte(got)) {