Emit events attached to node when snapshot retention fails

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

Author: brandond

pkg/etcd/snapshot.go

@@ -36,6 +36,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/util/retry"
@@ -347,10 +348,10 @@ func (e *ETCD) snapshot(ctx context.Context) (_ *managed.SnapshotResult, rerr er
 
 		// Snapshot retention may prune some files before returning an error. Failing to prune is not fatal.
 		deleted, err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir)
+		res.Deleted = append(res.Deleted, deleted...)
 		if err != nil {
-			logrus.Warnf("Failed to apply local snapshot retention policy: %v", err)
+			e.warningEventf("ETDCSnapshotRetentionFailedLocal", "Failed to apply local snapshot retention policy: %v", err)
 		}
-		res.Deleted = append(res.Deleted, deleted...)
 
 		if e.config.EtcdS3 != nil {
 			s3Start := time.Now()
@@ -390,7 +391,7 @@ func (e *ETCD) snapshot(ctx context.Context) (_ *managed.SnapshotResult, rerr er
 				deleted, err := s3client.SnapshotRetention(ctx, e.config.EtcdSnapshotName)
 				res.Deleted = append(res.Deleted, deleted...)
 				if err != nil {
-					logrus.Warnf("Failed to apply s3 snapshot retention policy: %v", err)
+					e.warningEventf("ETCDSnapshotRetentionFailedS3", "Failed to apply S3 snapshot retention policy: %v", err)
 				}
 			}
 			// sf is either s3 snapshot metadata, or s3 init/upload failure record.
@@ -650,7 +651,7 @@ func (e *ETCD) addSnapshotData(sf snapshot.File) error {
 			created, err = snapshots.Create(esf)
 			if err == nil {
 				// Only emit an event for the snapshot when creating the resource
-				e.emitEvent(created)
+				e.snapshotEvent(created)
 			}
 		} else if !equality.Semantic.DeepEqual(existing, esf) {
 			_, err = snapshots.Update(esf)
@@ -669,7 +670,8 @@ func generateETCDSnapshotFileConfigMapKey(esf k3s.ETCDSnapshotFile) string {
 	return "local-" + name
 }
 
-func (e *ETCD) emitEvent(esf *k3s.ETCDSnapshotFile) {
+// snapshotEvent emits an Event attached to the EtcdSnapshotFile resource
+func (e *ETCD) snapshotEvent(esf *k3s.ETCDSnapshotFile) {
 	switch {
 	case e.config.Runtime.Event == nil:
 	case !esf.DeletionTimestamp.IsZero():
@@ -685,6 +687,23 @@ func (e *ETCD) emitEvent(esf *k3s.ETCDSnapshotFile) {
 	}
 }
 
+// warningEventf emits a warning Event attached to the Node resource,
+// or directly logs a warning if the event recorder is not available.
+func (e *ETCD) warningEventf(reason, messageFmt string, args ...any) {
+	nodeName := os.Getenv("NODE_NAME")
+	if nodeName != "" && e.config.Runtime.Event != nil {
+		nodeRef := &v1.ObjectReference{
+			Kind:      "Node",
+			Name:      nodeName,
+			UID:       types.UID(nodeName),
+			Namespace: "",
+		}
+		e.config.Runtime.Event.Eventf(nodeRef, v1.EventTypeWarning, reason, messageFmt, args...)
+	} else {
+		logrus.Warnf(messageFmt, args...)
+	}
+}
+
 // ReconcileSnapshotData reconciles snapshot data in the ETCDSnapshotFile resources.
 // It will reconcile snapshot data from disk locally always, and if S3 is enabled, will attempt to
 // list S3 snapshots and reconcile snapshots from S3.

pkg/etcd/snapshot_controller.go

@@ -166,7 +166,7 @@ func (e *etcdSnapshotHandler) onRemove(key string, esf *k3s.ETCDSnapshotFile) (*
 			return nil, errors.WithMessage(err, "failed to remove snapshot from ConfigMap")
 		}
 	}
-	e.etcd.emitEvent(esf)
+	e.etcd.snapshotEvent(esf)
 	return nil, nil
 }