feat(controller): migrate instance event emission to the supported events API

The InstanceReconciler emitted quota and referenced-data events through the
deprecated core/v1 event recorder. Move to the events.k8s.io/v1 recorder so
Instance events keep flowing on the supported API surface, with each event now
carrying a machine-readable action (the controller operation) alongside its
existing reason (the outcome). Operators see the same Warning/Normal events in
`kubectl describe instance`, now backed by events.k8s.io.

All emit sites funnel through the emitEvent helper, which passes the message as
a "%s" argument so backend error text containing a literal '%' is not
format-expanded, and truncates the note to the 1024-character server-side limit
so oversized notes are not silently rejected. The controller RBAC role is
regenerated to grant events.k8s.io instead of the core events group.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: scotwells

config/components/controller_rbac/role.yaml

@@ -17,13 +17,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
 - apiGroups:
   - ""
   resources:
@@ -63,6 +56,13 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - networking.datumapis.com
   resources:

internal/controller/instance_controller.go

@@ -20,7 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/events"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
@@ -92,6 +92,23 @@ const (
 	reasonNetworkFailedToCreate = "NetworkFailedToCreate"
 )
 
+// Event action strings name the controller operation an event describes. The
+// events.k8s.io API separates the machine-readable action (what the controller
+// was doing, UpperCamelCase) from the reason (the outcome), so the reason
+// constants carry the failure mode while these name the operation.
+const (
+	eventActionResolvingReferencedData = "ResolvingReferencedData"
+	eventActionRemovingSchedulingGate  = "RemovingSchedulingGate"
+	eventActionClaimingQuota           = "ClaimingQuota"
+	eventActionReleasingQuota          = "ReleasingQuota"
+)
+
+// maxEventNoteLen bounds an event note's length. events.k8s.io/v1 rejects notes
+// longer than 1024 characters server-side, and the awaiting-propagation note
+// joins an unbounded list of missing companions, so the note is truncated
+// before emission to keep events from being silently dropped.
+const maxEventNoteLen = 1024
+
 // instanceTypeD1Standard2 is the platform instance type name for the
 // 1 vCPU / 2 GiB size used as the catalog baseline for quota accounting.
 const instanceTypeD1Standard2 = "datumcloud/d1-standard-2"
@@ -190,7 +207,7 @@ type InstanceReconciler struct {
 	edgeClusterName    string
 	// recorder emits Kubernetes events on the Instance object for quota failure
 	// modes so operators can diagnose issues via `kubectl describe`.
-	recorder record.EventRecorder
+	recorder events.EventRecorder
 	// projectIDForInstance derives the Milo project ID used for quota
 	// ResourceClaim management. In Milo mode it returns string(clusterName); in
 	// single-cell mode it reads the upstream-cluster-name label from the edge
@@ -224,7 +241,7 @@ type InstanceReconciler struct {
 // +kubebuilder:rbac:groups=compute.datumapis.com,resources=instances/finalizers,verbs=update
 // +kubebuilder:rbac:groups=quota.miloapis.com,resources=resourceclaims,verbs=get;list;watch;create;delete
 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get
-// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
 
 //nolint:gocyclo // conditions are reconciled, persisted, then returned as errors; the ordered pipeline is inherently branchy
 func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (_ ctrl.Result, err error) {
@@ -588,7 +605,8 @@ func (r *InstanceReconciler) reconcileReferencedDataCondition(
 			computev1alpha.ReferencedDataReasonAwaitingPropagation, msg)
 		if prevReason != computev1alpha.ReferencedDataReasonAwaitingPropagation {
 			r.emitEvent(instance, corev1.EventTypeWarning,
-				computev1alpha.ReferencedDataReasonAwaitingPropagation, msg)
+				computev1alpha.ReferencedDataReasonAwaitingPropagation,
+				eventActionResolvingReferencedData, msg)
 		}
 		return referencedDataResult{conditionChanged: changed}, 0, nil
 	}
@@ -902,16 +920,23 @@ func (r *InstanceReconciler) observeGateWaitDuration(instance *computev1alpha.In
 func (r *InstanceReconciler) emitReferencedDataClearedEvent(instance *computev1alpha.Instance) {
 	r.emitEvent(instance, corev1.EventTypeNormal,
 		computev1alpha.ReferencedDataReasonReady,
+		eventActionRemovingSchedulingGate,
 		"All referenced companion ConfigMaps/Secrets are present; ReferencedData gate cleared")
 }
 
 // emitEvent emits a Kubernetes event if a recorder is available. Guard against
 // a nil recorder so that unit tests that don't wire up a recorder don't panic.
-func (r *InstanceReconciler) emitEvent(obj *computev1alpha.Instance, eventType, reason, message string) {
+// The pre-built message is passed as the "%s" argument, never as the note format
+// string itself, because the events API treats the note as a printf format and
+// the quota messages embed backend error text that can contain literal '%'.
+func (r *InstanceReconciler) emitEvent(obj *computev1alpha.Instance, eventType, reason, action, message string) {
 	if r.recorder == nil {
 		return
 	}
-	r.recorder.Event(obj, eventType, reason, message)
+	if len(message) > maxEventNoteLen {
+		message = strings.ToValidUTF8(message[:maxEventNoteLen-3], "") + "..."
+	}
+	r.recorder.Eventf(obj, nil, eventType, reason, action, "%s", message)
 }
 
 // reconcileDeletion handles quota-claim cleanup when an Instance is being
@@ -936,8 +961,9 @@ func (r *InstanceReconciler) reconcileDeletion(ctx context.Context, cl client.Cl
 			// claim will count against project budget until Milo's TTL/GC removes it.
 			log.FromContext(ctx).Error(err, "project identity unresolvable during deletion; ResourceClaim may be orphaned — budget leak possible",
 				"instance", instance.Name, "namespace", instance.Namespace)
-			r.recorder.Event(instance, corev1.EventTypeWarning,
+			r.emitEvent(instance, corev1.EventTypeWarning,
 				"QuotaClaimOrphaned",
+				eventActionReleasingQuota,
 				"Skipping ResourceClaim cleanup: project identity could not be resolved; claim may be orphaned in Milo project control plane")
 			quotametrics.ClaimOrphanedTotal.Inc()
 		}
@@ -1055,8 +1081,9 @@ func (r *InstanceReconciler) reconcileQuotaCondition(ctx context.Context, cluste
 			Message:            "ResourceClaim is pending: no AllowanceBucket configured for this project",
 			ObservedGeneration: instance.Generation,
 		})
-		r.recorder.Event(instance, corev1.EventTypeWarning,
+		r.emitEvent(instance, corev1.EventTypeWarning,
 			computev1alpha.InstanceQuotaGrantedReasonNoBudget,
+			eventActionClaimingQuota,
 			"ResourceClaim pending: no AllowanceBucket configured for this project")
 		quotametrics.EvalFailuresTotal.WithLabelValues(quotametrics.ReasonNoBudget).Inc()
 		return changed, claimErr
@@ -1281,8 +1308,8 @@ func (r *InstanceReconciler) reconcileQuotaClaim(ctx context.Context, clusterNam
 		// structured condition + warning event + error return, instead of
 		// silently parking the instance at PendingEvaluation.
 		msg := fmt.Sprintf("Could not resolve project ID: %v", err)
-		r.recorder.Event(instance, corev1.EventTypeWarning,
-			computev1alpha.InstanceQuotaGrantedReasonProjectIDUnresolvable, msg)
+		r.emitEvent(instance, corev1.EventTypeWarning,
+			computev1alpha.InstanceQuotaGrantedReasonProjectIDUnresolvable, eventActionClaimingQuota, msg)
 		quotametrics.EvalFailuresTotal.WithLabelValues(quotametrics.ReasonProjectIDUnresolvable).Inc()
 		return &metav1.Condition{
 			Type:    computev1alpha.InstanceQuotaGranted,
@@ -1295,8 +1322,8 @@ func (r *InstanceReconciler) reconcileQuotaClaim(ctx context.Context, clusterNam
 	projectClient, err := r.quotaClientManager.ClientForProject(ctx, projectID, r.scheme)
 	if err != nil {
 		msg := fmt.Sprintf("Failed to build quota client for project %q: %v", projectID, err)
-		r.recorder.Event(instance, corev1.EventTypeWarning,
-			computev1alpha.InstanceQuotaGrantedReasonBackendUnavailable, msg)
+		r.emitEvent(instance, corev1.EventTypeWarning,
+			computev1alpha.InstanceQuotaGrantedReasonBackendUnavailable, eventActionClaimingQuota, msg)
 		quotametrics.EvalFailuresTotal.WithLabelValues(quotametrics.ReasonBackendUnavailable).Inc()
 		return &metav1.Condition{
 			Type:    computev1alpha.InstanceQuotaGranted,
@@ -1309,8 +1336,8 @@ func (r *InstanceReconciler) reconcileQuotaClaim(ctx context.Context, clusterNam
 	claimNamespace, err := r.resolveProjectNamespace(ctx, clusterName, instance)
 	if err != nil {
 		msg := fmt.Sprintf("Could not resolve project namespace: %v", err)
-		r.recorder.Event(instance, corev1.EventTypeWarning,
-			computev1alpha.InstanceQuotaGrantedReasonProjectIDUnresolvable, msg)
+		r.emitEvent(instance, corev1.EventTypeWarning,
+			computev1alpha.InstanceQuotaGrantedReasonProjectIDUnresolvable, eventActionClaimingQuota, msg)
 		quotametrics.EvalFailuresTotal.WithLabelValues(quotametrics.ReasonProjectIDUnresolvable).Inc()
 		return &metav1.Condition{
 			Type:    computev1alpha.InstanceQuotaGranted,
@@ -1381,8 +1408,8 @@ func (r *InstanceReconciler) reconcileQuotaClaim(ctx context.Context, clusterNam
 		}
 		// GET itself failed — treat as backend unavailable.
 		msg := fmt.Sprintf("Quota backend unreachable getting ResourceClaim: %v", err)
-		r.recorder.Event(instance, corev1.EventTypeWarning,
-			computev1alpha.InstanceQuotaGrantedReasonBackendUnavailable, msg)
+		r.emitEvent(instance, corev1.EventTypeWarning,
+			computev1alpha.InstanceQuotaGrantedReasonBackendUnavailable, eventActionClaimingQuota, msg)
 		quotametrics.EvalFailuresTotal.WithLabelValues(quotametrics.ReasonBackendUnavailable).Inc()
 		return &metav1.Condition{
 			Type:    computev1alpha.InstanceQuotaGranted,
@@ -1431,7 +1458,7 @@ func (r *InstanceReconciler) classifyCreateError(
 		msg = fmt.Sprintf("Quota backend unreachable creating ResourceClaim: %v", err)
 	}
 
-	r.recorder.Event(instance, corev1.EventTypeWarning, reason, msg)
+	r.emitEvent(instance, corev1.EventTypeWarning, reason, eventActionClaimingQuota, msg)
 	quotametrics.EvalFailuresTotal.WithLabelValues(metricLabel).Inc()
 	return &metav1.Condition{
 		Type:    computev1alpha.InstanceQuotaGranted,
@@ -1854,10 +1881,7 @@ func (r *InstanceReconciler) SetupWithManager(
 ) error {
 	r.mgr = mgr
 	r.scheme = mgr.GetLocalManager().GetScheme()
-	//nolint:staticcheck // GetEventRecorder (new events API) has an incompatible Eventf
-	// signature (requires related object + action args) that would require migrating
-	// all emit sites. GetEventRecorderFor remains correct; migration is deferred.
-	r.recorder = mgr.GetLocalManager().GetEventRecorderFor("instance-controller")
+	r.recorder = mgr.GetLocalManager().GetEventRecorder("instance-controller")
 	r.edgeClusterName = edgeClusterName
 	r.projectIDForInstance = projectIDForInstance
 	r.projectNamespaceForInstance = projectNamespaceForInstance

internal/controller/instance_controller_test.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -16,7 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
@@ -1231,7 +1232,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 	newReconcilerWithInterceptor := func(
 		t *testing.T,
 		funcs interceptor.Funcs,
-		fakeRecorder *record.FakeRecorder,
+		fakeRecorder *capturingEventRecorder,
 	) (*InstanceReconciler, client.Client) {
 		t.Helper()
 		s := newTestScheme(t)
@@ -1279,7 +1280,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 	}
 
 	t.Run("FM-2: backend unreachable sets QuotaBackendUnavailable", func(t *testing.T) {
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 		r, projectClient := newReconcilerWithInterceptor(t, interceptor.Funcs{
 			Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error {
 				return fmt.Errorf("connection refused")
@@ -1306,13 +1307,22 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 		default:
 			t.Error("expected a Warning event for backend unavailable, got none")
 		}
+
+		// The event carries the quota-claim action and references the Instance.
+		// The apiserver rejects an events.k8s.io event with an empty action, so a
+		// silently-empty action would 422-drop in production while passing CI.
+		last := fakeRecorder.LastRecorded()
+		require.NotNil(t, last)
+		assert.Equal(t, eventActionClaimingQuota, last.Action)
+		assert.NotEmpty(t, last.Action)
+		assert.NotNil(t, last.Regarding)
 	})
 
 	// FM-4/FM-5: 404 on Create maps to NamespaceNotFound when the claim namespace
 	// is known (the more common case for project-exists-but-namespace-absent), and
 	// to ProjectNotFound when the namespace itself is empty (project CP path missing).
 	t.Run("FM-5: 404 on Create with known namespace sets QuotaNamespaceNotFound", func(t *testing.T) {
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 		notFoundErr := apierrors.NewNotFound(
 			schema.GroupResource{Group: testQuotaAPIGroup, Resource: testQuotaResource}, "claim")
 		r, projectClient := newReconcilerWithInterceptor(t, interceptor.Funcs{
@@ -1347,7 +1357,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 	})
 
 	t.Run("FM-6: 403 on Create sets QuotaMisconfigured", func(t *testing.T) {
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 		forbiddenErr := apierrors.NewForbidden(
 			schema.GroupResource{Group: testQuotaAPIGroup, Resource: testQuotaResource}, "claim",
 			fmt.Errorf("ResourceRegistration not found"))
@@ -1384,7 +1394,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 
 	t.Run("FM-7: claim pending with no budget sets QuotaNoBudget", func(t *testing.T) {
 		s := newTestScheme(t)
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 
 		claimName := instanceQuotaClaimNamePrefix + testInstance
 		pendingClaim := &quotav1alpha1.ResourceClaim{
@@ -1492,7 +1502,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 			scheme:             s,
 			quotaClientManager: nil, // explicitly disabled
 			edgeClusterName:    testEdgeClusterName,
-			recorder:           record.NewFakeRecorder(10),
+			recorder:           newCapturingEventRecorder(10),
 			projectIDForInstance: func(_ context.Context, cn multicluster.ClusterName, _ *computev1alpha.Instance) (string, error) {
 				return string(cn), nil
 			},
@@ -1516,7 +1526,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 
 	t.Run("observedGeneration guard: stale True condition does not remove gate for new generation", func(t *testing.T) {
 		s := newTestScheme(t)
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 
 		// Instance at generation 2 with a stale QuotaGranted=True from generation 1.
 		instance := makeInstance()
@@ -1612,7 +1622,7 @@ func TestReconcileQuotaFailureModes(t *testing.T) {
 
 	t.Run("FM-1: missing identity label sets ProjectIDUnresolvable and errors", func(t *testing.T) {
 		s := newTestScheme(t)
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 
 		projectClient := fake.NewClientBuilder().
 			WithScheme(s).
@@ -1713,7 +1723,7 @@ func TestReconcileDeletionProjectIdentity(t *testing.T) {
 		}
 	}
 
-	newReconciler := func(t *testing.T, projectIDFn InstanceProjectIDFunc, rec record.EventRecorder) (*InstanceReconciler, client.Client, client.Client) {
+	newReconciler := func(t *testing.T, projectIDFn InstanceProjectIDFunc, rec events.EventRecorder) (*InstanceReconciler, client.Client, client.Client) {
 		t.Helper()
 		s := newTestScheme(t)
 		projectClient := fake.NewClientBuilder().
@@ -1752,7 +1762,7 @@ func TestReconcileDeletionProjectIdentity(t *testing.T) {
 	}
 
 	t.Run("unresolvable identity: deletion proceeds, claim cleanup skipped", func(t *testing.T) {
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 		identityErr := fmt.Errorf("edge namespace %q is missing label %q: %w",
 			namespace, downstreamclient.UpstreamOwnerClusterNameLabel, errProjectIdentityUnresolvable)
 		r, projectClient, quotaClient := newReconciler(t,
@@ -1784,10 +1794,15 @@ func TestReconcileDeletionProjectIdentity(t *testing.T) {
 		default:
 			t.Error("expected a QuotaClaimOrphaned event, got none")
 		}
+
+		// The deletion-path event names the quota-release action.
+		last := fakeRecorder.LastRecorded()
+		require.NotNil(t, last)
+		assert.Equal(t, eventActionReleasingQuota, last.Action)
 	})
 
 	t.Run("transient resolution failure: reconcile errors and retries", func(t *testing.T) {
-		fakeRecorder := record.NewFakeRecorder(10)
+		fakeRecorder := newCapturingEventRecorder(10)
 		r, projectClient, quotaClient := newReconciler(t,
 			func(_ context.Context, _ multicluster.ClusterName, _ *computev1alpha.Instance) (string, error) {
 				return "", fmt.Errorf("connection refused")
@@ -2433,7 +2448,7 @@ func TestReconcileQuotaClaim_RequestsIncludeVCPUsAndMemory(t *testing.T) {
 		projectIDForInstance: func(_ context.Context, cn multicluster.ClusterName, _ *computev1alpha.Instance) (string, error) {
 			return string(cn), nil
 		},
-		recorder: &record.FakeRecorder{},
+		recorder: &capturingEventRecorder{},
 	}
 	r.finalizers = finalizer.NewFinalizers()
 	require.NoError(t, r.finalizers.Register(instanceControllerFinalizer, r))
@@ -2772,3 +2787,43 @@ func TestInstanceBlockingReasonPriority(t *testing.T) {
 		})
 	}
 }
+
+// TestEmitEvent exercises the three load-bearing behaviors of the emitEvent
+// helper directly: the nil-recorder guard, the "%s" indirection that keeps a
+// literal '%' in a message from being format-expanded, and the note truncation
+// that keeps events.k8s.io/v1 from rejecting an oversized note server-side.
+func TestEmitEvent(t *testing.T) {
+	obj := &computev1alpha.Instance{
+		ObjectMeta: metav1.ObjectMeta{Name: "emit-test-instance", Namespace: "default"},
+	}
+
+	t.Run("nil recorder does not panic", func(t *testing.T) {
+		r := &InstanceReconciler{}
+		require.NotPanics(t, func() {
+			r.emitEvent(obj, corev1.EventTypeWarning, "SomeReason", eventActionClaimingQuota, "msg")
+		})
+	})
+
+	t.Run("literal % in message is not format-expanded", func(t *testing.T) {
+		rec := newCapturingEventRecorder(1)
+		r := &InstanceReconciler{recorder: rec}
+		r.emitEvent(obj, corev1.EventTypeWarning, "SomeReason", eventActionClaimingQuota,
+			"connection refused: path %2Fapi got 50% errors")
+		last := rec.LastRecorded()
+		require.NotNil(t, last)
+		assert.Contains(t, last.Note, "%2Fapi")
+		assert.Contains(t, last.Note, "50%")
+		assert.NotContains(t, last.Note, "(MISSING)")
+	})
+
+	t.Run("note truncated at maxEventNoteLen", func(t *testing.T) {
+		rec := newCapturingEventRecorder(1)
+		r := &InstanceReconciler{recorder: rec}
+		r.emitEvent(obj, corev1.EventTypeWarning, "SomeReason", eventActionClaimingQuota,
+			strings.Repeat("x", 2000))
+		last := rec.LastRecorded()
+		require.NotNil(t, last)
+		assert.LessOrEqual(t, len(last.Note), maxEventNoteLen)
+		assert.True(t, strings.HasSuffix(last.Note, "..."))
+	})
+}