HYPERFLEET-856 - fix: address PR review findings

- Aggregate alert exprs with max by(resource_type) to deduplicate across pods
- Template alert description durations from values instead of hard-coding
- Add context timeout (5s) to TerminatingCollector.Collect DB queries
- Record nodepool termination metrics on cluster cascade delete
- Validate DeletionStuckThreshold > 0 in config loader

Author: rafabene

charts/templates/prometheusrule.yaml

@@ -14,25 +14,25 @@ spec:
     - name: hyperfleet-api-deletion
       rules:
         - alert: HyperFleetResourceDeletionStuck
-          expr: hyperfleet_api_resource_terminating_stuck > 0
+          expr: max by (resource_type)(hyperfleet_api_resource_terminating_stuck) > 0
           for: {{ .Values.prometheusRule.rules.deletionStuck.for | default "5m" }}
           labels:
             severity: warning
           annotations:
             summary: "HyperFleet resources stuck in Terminating state"
             description: >-
               {{ "{{ $value }}" }} {{ "{{ $labels.resource_type }}" }} resource(s) have been in
-              Terminating state for more than 30 minutes.
+              Terminating state for more than {{ .Values.prometheusRule.rules.deletionStuck.for | default "5m" }}.
             runbook_url: {{ .Values.prometheusRule.rules.deletionStuck.runbookUrl | default "" | quote }}
         - alert: HyperFleetResourceDeletionTimeout
-          expr: hyperfleet_api_resource_terminating_stuck > 0
+          expr: max by (resource_type)(hyperfleet_api_resource_terminating_stuck) > 0
           for: {{ .Values.prometheusRule.rules.deletionTimeout.for | default "30m" }}
           labels:
             severity: critical
           annotations:
             summary: "HyperFleet resources timed out in Terminating state"
             description: >-
               {{ "{{ $value }}" }} {{ "{{ $labels.resource_type }}" }} resource(s) have been in
-              Terminating state for more than 1 hour. Immediate investigation required.
+              Terminating state for more than {{ .Values.prometheusRule.rules.deletionTimeout.for | default "30m" }}. Immediate investigation required.
             runbook_url: {{ .Values.prometheusRule.rules.deletionTimeout.runbookUrl | default "" | quote }}
 {{- end }}

pkg/config/loader.go

@@ -185,6 +185,9 @@ func (l *ConfigLoader) validateConfig(config *ApplicationConfig) error {
 		if valErr := config.Metrics.TLS.Validate(); valErr != nil {
 			return fmt.Errorf("metrics TLS validation failed: %w", valErr)
 		}
+		if valErr := config.Metrics.Validate(); valErr != nil {
+			return fmt.Errorf("metrics config validation failed: %w", valErr)
+		}
 		return nil
 	}
 

pkg/config/metrics.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"net"
 	"strconv"
 	"time"
@@ -30,6 +31,14 @@ func NewMetricsConfig() *MetricsConfig {
 	}
 }
 
+// Validate validates MetricsConfig fields that struct tags cannot enforce
+func (m *MetricsConfig) Validate() error {
+	if m.DeletionStuckThreshold <= 0 {
+		return fmt.Errorf("DeletionStuckThreshold must be positive, got %v", m.DeletionStuckThreshold)
+	}
+	return nil
+}
+
 // ============================================================
 // Convenience Accessor Methods
 // ============================================================

pkg/metrics/deletion.go

@@ -17,6 +17,7 @@ limitations under the License.
 package metrics
 
 import (
+	"context"
 	"database/sql"
 	"log/slog"
 	"sync"
@@ -98,16 +99,20 @@ func ResetMetrics() {
 // TerminatingCollector implements prometheus.Collector to report the number of
 // resources stuck in Terminating state beyond a configurable threshold.
 // It queries the database on each Prometheus scrape.
+const defaultQueryTimeout = 5 * time.Second
+
 type TerminatingCollector struct {
 	stuckDesc      *prometheus.Desc
 	db             *sql.DB
 	stuckThreshold time.Duration
+	queryTimeout   time.Duration
 }
 
 func NewTerminatingCollector(db *sql.DB, stuckThreshold time.Duration) *TerminatingCollector {
 	return &TerminatingCollector{
 		db:             db,
 		stuckThreshold: stuckThreshold,
+		queryTimeout:   defaultQueryTimeout,
 		stuckDesc: prometheus.NewDesc(
 			metricsSubsystem+"_resource_terminating_stuck",
 			"Number of resources in Terminating state beyond the stuck threshold.",
@@ -136,11 +141,14 @@ func (c *TerminatingCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), c.queryTimeout)
+	defer cancel()
+
 	threshold := time.Now().UTC().Add(-c.stuckThreshold)
 
 	for _, q := range stuckQueries {
 		var count int64
-		row := c.db.QueryRow(q.query, threshold) //nolint:gosec // table names are compile-time constants
+		row := c.db.QueryRowContext(ctx, q.query, threshold) //nolint:gosec // table names are compile-time constants
 		if err := row.Scan(&count); err != nil {
 			slog.Error("Failed to query terminating resources",
 				"resource_type", q.resourceType,

pkg/services/cluster.go

@@ -140,11 +140,15 @@ func (s *sqlClusterService) SoftDelete(ctx context.Context, id string) (*api.Clu
 		return nil, svcErr
 	}
 
-	// Update status for all cascade-deleted nodepools so their Ready condition reflects the generation bump.
 	nodePools, err := s.nodePoolDao.FindSoftDeletedByOwner(ctx, id)
 	if err != nil {
 		return nil, errors.GeneralError("Failed to fetch cascade-deleted nodepools: %s", err)
 	}
+
+	for range nodePools {
+		metrics.RecordTerminating("nodepool")
+	}
+
 	if svcErr := batchUpdateNodePoolStatusesFromAdapters(
 		ctx,
 		nodePools,