feat: label NodeClaim termination and lifetime histograms with instance and capacity type

pkg/controllers/nodeclaim/lifecycle/controller.go

@@ -258,10 +258,7 @@ func (c *Controller) finalize(ctx context.Context, nodeClaim *v1.NodeClaim) (rec
 		if err := c.deleteUnregisteredNodes(ctx, nodeClaim); err != nil {
 			return reconcile.Result{}, err
 		}
-		InstanceTerminationDurationSeconds.Observe(time.Since(nodeClaim.StatusConditions().Get(v1.ConditionTypeInstanceTerminating).LastTransitionTime.Time).Seconds(), map[string]string{
-			metrics.NodePoolLabel: nodeClaim.Labels[v1.NodePoolLabelKey],
-			causeLabel:            terminationCause(nodeClaim),
-		})
+		InstanceTerminationDurationSeconds.Observe(time.Since(nodeClaim.StatusConditions().Get(v1.ConditionTypeInstanceTerminating).LastTransitionTime.Time).Seconds(), terminationLabels(nodeClaim))
 	}
 	stored := nodeClaim.DeepCopy() // The NodeClaim may have been modified in the EnsureTerminated function
 	controllerutil.RemoveFinalizer(nodeClaim, v1.TerminationFinalizer)
@@ -277,10 +274,7 @@ func (c *Controller) finalize(ctx context.Context, nodeClaim *v1.NodeClaim) (rec
 			return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("removing termination finalizer, %w", err))
 		}
 		log.FromContext(ctx).Info("deleted nodeclaim")
-		NodeClaimTerminationDurationSeconds.Observe(time.Since(stored.DeletionTimestamp.Time).Seconds(), map[string]string{
-			metrics.NodePoolLabel: nodeClaim.Labels[v1.NodePoolLabelKey],
-			causeLabel:            terminationCause(nodeClaim),
-		})
+		NodeClaimTerminationDurationSeconds.Observe(time.Since(stored.DeletionTimestamp.Time).Seconds(), terminationLabels(nodeClaim))
 		metrics.NodeClaimsTerminatedTotal.Inc(map[string]string{
 			metrics.NodePoolLabel:     nodeClaim.Labels[v1.NodePoolLabelKey],
 			metrics.CapacityTypeLabel: nodeClaim.Labels[v1.CapacityTypeLabelKey],
@@ -321,12 +315,29 @@ func observeLifetime(nodeClaim *v1.NodeClaim) {
 	if nodeClaim.CreationTimestamp.IsZero() || nodeClaim.DeletionTimestamp.IsZero() {
 		return
 	}
-	NodeClaimLifetimeSeconds.Observe(nodeClaim.DeletionTimestamp.Sub(nodeClaim.CreationTimestamp.Time).Seconds(), map[string]string{
+	labels := terminationLabels(nodeClaim)
+	labels[originLabel] = lo.ValueOr(nodeClaim.Annotations, v1.NodeClaimReplacementOriginAnnotationKey, provisioningOrigin)
+	NodeClaimLifetimeSeconds.Observe(nodeClaim.DeletionTimestamp.Sub(nodeClaim.CreationTimestamp.Time).Seconds(), labels)
+}
+
+// terminationLabels names the NodePool, the instance and capacity type the NodeClaim launched with, and
+// the cause of its deletion, for the histograms recorded when a NodeClaim is torn down. The instance and
+// capacity type come from the labels the cloud provider stamped at launch, so a NodeClaim deleted before it
+// launched reports them as unknown rather than as an empty label.
+func terminationLabels(nodeClaim *v1.NodeClaim) map[string]string {
+	return map[string]string{
 		metrics.NodePoolLabel:     nodeClaim.Labels[v1.NodePoolLabelKey],
-		metrics.CapacityTypeLabel: nodeClaim.Labels[v1.CapacityTypeLabelKey],
-		originLabel:               lo.ValueOr(nodeClaim.Annotations, v1.NodeClaimReplacementOriginAnnotationKey, provisioningOrigin),
+		instanceTypeLabel:         labelOrUnknown(nodeClaim, corev1.LabelInstanceTypeStable),
+		metrics.CapacityTypeLabel: labelOrUnknown(nodeClaim, v1.CapacityTypeLabelKey),
 		causeLabel:                terminationCause(nodeClaim),
-	})
+	}
+}
+
+func labelOrUnknown(nodeClaim *v1.NodeClaim, key string) string {
+	if value := nodeClaim.Labels[key]; value != "" {
+		return value
+	}
+	return unknownLabelValue
 }
 
 // terminationCause names why a NodeClaim was deleted from what the NodeClaim itself records: the

pkg/controllers/nodeclaim/lifecycle/metrics.go

@@ -30,10 +30,10 @@ var InstanceTerminationDurationSeconds = opmetrics.NewPrometheusHistogram(
 		Namespace: metrics.Namespace,
 		Subsystem: metrics.NodeClaimSubsystem,
 		Name:      "instance_termination_duration_seconds",
-		Help:      "Duration of CloudProvider Instance termination in seconds, by NodePool and the cause of the deletion (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
+		Help:      "Duration of CloudProvider Instance termination in seconds, by NodePool, the instance and capacity type the NodeClaim launched with (unknown when it never launched), and the cause of the deletion (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
 		Buckets:   prometheus.ExponentialBuckets(1, 2, 11), //The threshold values generated here are 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024
 	},
-	[]string{metrics.NodePoolLabel, causeLabel},
+	[]string{metrics.NodePoolLabel, instanceTypeLabel, metrics.CapacityTypeLabel, causeLabel},
 )
 
 // NodeClaimLifetimeSeconds records how long each NodeClaim existed, from creation to the removal of its
@@ -47,15 +47,20 @@ var NodeClaimLifetimeSeconds = opmetrics.NewPrometheusHistogram(
 		Namespace: metrics.Namespace,
 		Subsystem: metrics.NodeClaimSubsystem,
 		Name:      "lifetime_seconds",
-		Help:      "Seconds from NodeClaim creation to deletion, by NodePool, capacity type, the origin that launched it (provisioning, or the replacement-origin annotation set by the disruption queue), and the cause of its termination (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
+		Help:      "Seconds from NodeClaim creation to deletion, by NodePool, the instance type it launched with (unknown when it never launched), capacity type, the origin that launched it (provisioning, or the replacement-origin annotation set by the disruption queue), and the cause of its termination (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
 		Buckets:   metrics.NodeLifetimeBuckets(),
 	},
-	[]string{metrics.NodePoolLabel, metrics.CapacityTypeLabel, originLabel, causeLabel},
+	[]string{metrics.NodePoolLabel, instanceTypeLabel, metrics.CapacityTypeLabel, originLabel, causeLabel},
 )
 
 const (
-	originLabel = "origin"
-	causeLabel  = "cause"
+	instanceTypeLabel = "instance_type"
+	originLabel       = "origin"
+	causeLabel        = "cause"
+
+	// unknownLabelValue stands in for an instance or capacity type the NodeClaim's labels do not carry,
+	// which is the case for a NodeClaim deleted before the cloud provider ever launched it.
+	unknownLabelValue = "unknown"
 
 	// provisioningOrigin is the origin of every NodeClaim not launched as a disruption replacement.
 	provisioningOrigin = "provisioning"
@@ -74,7 +79,7 @@ var NodeClaimTerminationDurationSeconds = opmetrics.NewPrometheusHistogram(
 		Namespace: metrics.Namespace,
 		Subsystem: metrics.NodeClaimSubsystem,
 		Name:      "termination_duration_seconds",
-		Help:      "Duration of NodeClaim termination in seconds, by NodePool and the cause of the deletion (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
+		Help:      "Duration of NodeClaim termination in seconds, by NodePool, the instance and capacity type the NodeClaim launched with (unknown when it never launched), and the cause of the deletion (a voluntary disruption reason, the termination-cause annotation such as cloud_interrupted, never_initialized, or other).",
 		Buckets:   prometheus.ExponentialBuckets(1, 2, 12)}, //The threshold values generated here are 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024. 2048
-	[]string{metrics.NodePoolLabel, causeLabel},
+	[]string{metrics.NodePoolLabel, instanceTypeLabel, metrics.CapacityTypeLabel, causeLabel},
 )

pkg/controllers/nodeclaim/lifecycle/termination_test.go

@@ -199,20 +199,43 @@ var _ = Describe("Termination", func() {
 		ExpectObjectReconciled(ctx, env.Client, nodeClaimController, nodeClaim)
 		ExpectNotFound(ctx, env.Client, nodeClaim)
 
-		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_instance_termination_duration_seconds", 1, map[string]string{
-			metrics.NodePoolLabel: nodePool.Name,
-			"cause":               v1.NodeClaimTerminationCauseCloudInterrupted,
-		})
-		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_termination_duration_seconds", 1, map[string]string{
-			metrics.NodePoolLabel: nodePool.Name,
-			"cause":               v1.NodeClaimTerminationCauseCloudInterrupted,
-		})
-		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_lifetime_seconds", 1, map[string]string{
+		// The instance and capacity type the fake cloud provider stamped at launch travel onto every
+		// termination histogram, so interruptions can be broken down by what was actually running.
+		Expect(nodeClaim.Labels[corev1.LabelInstanceTypeStable]).ToNot(BeEmpty())
+		Expect(nodeClaim.Labels[v1.CapacityTypeLabelKey]).ToNot(BeEmpty())
+		launchedLabels := map[string]string{
 			metrics.NodePoolLabel:     nodePool.Name,
+			"instance_type":           nodeClaim.Labels[corev1.LabelInstanceTypeStable],
 			metrics.CapacityTypeLabel: nodeClaim.Labels[v1.CapacityTypeLabelKey],
-			"origin":                  "provisioning",
 			"cause":                   v1.NodeClaimTerminationCauseCloudInterrupted,
-		})
+		}
+		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_instance_termination_duration_seconds", 1, launchedLabels)
+		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_termination_duration_seconds", 1, launchedLabels)
+		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_lifetime_seconds", 1, lo.Assign(launchedLabels, map[string]string{
+			"origin": "provisioning",
+		}))
+	})
+	It("reports the instance and capacity type as unknown for a NodeClaim deleted before it launched", func() {
+		lifecycle.NodeClaimLifetimeSeconds.Reset()
+		lifecycle.NodeClaimTerminationDurationSeconds.Reset()
+		ExpectApplied(ctx, env.Client, nodePool, nodeClaim)
+		Expect(nodeClaim.Labels).ToNot(HaveKey(corev1.LabelInstanceTypeStable))
+		Expect(nodeClaim.Labels).ToNot(HaveKey(v1.CapacityTypeLabelKey))
+
+		Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
+		ExpectObjectReconciled(ctx, env.Client, nodeClaimController, nodeClaim)
+		ExpectNotFound(ctx, env.Client, nodeClaim)
+
+		unlaunchedLabels := map[string]string{
+			metrics.NodePoolLabel:     nodePool.Name,
+			"instance_type":           "unknown",
+			metrics.CapacityTypeLabel: "unknown",
+			"cause":                   "never_initialized",
+		}
+		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_termination_duration_seconds", 1, unlaunchedLabels)
+		ExpectMetricHistogramSampleCountValue("karpenter_nodeclaims_lifetime_seconds", 1, lo.Assign(unlaunchedLabels, map[string]string{
+			"origin": "provisioning",
+		}))
 	})
 	It("shouldn't mark the root condition of the NodeClaim as unknown when setting the Termination condition", func() {
 		ExpectApplied(ctx, env.Client, nodePool, nodeClaim)