Add dashboard panels and alert for scan monitoring

This commit adds metric definitions to the Grafana dashboard for all new
Prometheus metrics:
- FULL_SCAN_ELAPSED, SCAN_COUNT (conductor)
- BUCKET_PROCESSOR_SCAN_START_TIME, BUCKET_PROCESSOR_BUCKETS_COUNT
we're also adding four new dashboard panels in the Lifecycle Conductor section:
- Conductor Full Scan Elapsed Time
- Conductor Scan Counts (buckets, lifecycle buckets, workflows)
- Bucket Processor Scan Progress (per-pod scan-in-progress state)
- Bucket Processor Lifecycle Buckets (per-pod processed count)
A LifecycleScanDurationHigh warning alert that fires when
a conductor scan exceeds 1 hour was also added.

Issue: BB-740

Author: benzekrimaha

monitoring/lifecycle/alerts.yaml

@@ -82,6 +82,21 @@ groups:
         {{ ${lifecycle_latency_critical_threshold} | humanizeDuration }} ago.
       summary: "Lifecycle scan not executed in time"
 
+  - alert: LifecycleScanDurationHigh
+    Expr: |
+      s3_lifecycle_conductor_full_scan_elapsed_seconds{
+        namespace="${namespace}", job="${job_lifecycle_producer}"
+      } > 3600
+    For: "0s"
+    Labels:
+      severity: warning
+    Annotations:
+      zenko_service: backbeat-lifecycle-producer
+      description: >-
+        The latest lifecycle conductor full scan took more than 1 hour
+        ({{ $value | humanizeDuration }}).
+      summary: "Lifecycle conductor full scan duration is high"
+
 - name: LifecycleBucketProcessor
   rules:
 

monitoring/lifecycle/dashboard.py

@@ -73,6 +73,26 @@ class Metrics:
         'status', job='${job_lifecycle_producer}', namespace='${namespace}',
     )
 
+    FULL_SCAN_ELAPSED = metrics.Metric(
+        's3_lifecycle_conductor_full_scan_elapsed_seconds',
+        job='${job_lifecycle_producer}', namespace='${namespace}',
+    )
+
+    SCAN_COUNT = metrics.Metric(
+        's3_lifecycle_conductor_scan_count',
+        'type', job='${job_lifecycle_producer}', namespace='${namespace}',
+    )
+
+    BUCKET_PROCESSOR_SCAN_START_TIME = metrics.Metric(
+        's3_lifecycle_bucket_processor_scan_start_time',
+        job='${job_lifecycle_bucket_processor}', namespace='${namespace}',
+    )
+
+    BUCKET_PROCESSOR_BUCKETS_COUNT = metrics.Metric(
+        's3_lifecycle_bucket_processor_buckets_count',
+        job='${job_lifecycle_bucket_processor}', namespace='${namespace}',
+    )
+
     S3_OPS = metrics.CounterMetric(
        's3_lifecycle_s3_operations_total',
        'origin', 'op', 'status', job=['$jobs'], namespace='${namespace}',
@@ -730,6 +750,70 @@ def color_override(name, color):
     ]
 )
 
+lifecycle_full_scan_elapsed = TimeSeries(
+    title="Conductor Full Scan Elapsed Time",
+    dataSource="${DS_PROMETHEUS}",
+    legendDisplayMode="hidden",
+    unit=UNITS.SECONDS,
+    targets=[
+        Target(
+            expr=Metrics.FULL_SCAN_ELAPSED(),
+            legendFormat='Elapsed',
+        ),
+    ],
+)
+
+lifecycle_scan_count = TimeSeries(
+    title="Conductor Scan Counts",
+    dataSource="${DS_PROMETHEUS}",
+    fillOpacity=20,
+    lineInterpolation="smooth",
+    targets=[
+        Target(
+            expr=Metrics.SCAN_COUNT(type='bucket'),
+            legendFormat='Buckets',
+        ),
+        Target(
+            expr=Metrics.SCAN_COUNT(type='lifecycle_bucket'),
+            legendFormat='Lifecycle Buckets',
+        ),
+        Target(
+            expr=Metrics.SCAN_COUNT(type='workflow'),
+            legendFormat='Workflows',
+        ),
+    ],
+)
+
+bucket_processor_scan_progress = TimeSeries(
+    title="Bucket Processor Scan Progress",
+    description="Shows whether each bucket processor is actively "
+        "processing a scan. Non-zero = scan in progress (value is "
+        "the scan start timestamp). When all pods reset to 0, the "
+        "end-to-end scan is complete.",
+    dataSource="${DS_PROMETHEUS}",
+    lineInterpolation="smooth",
+    targets=[
+        Target(
+            expr=Metrics.BUCKET_PROCESSOR_SCAN_START_TIME() + ' > 0',
+            legendFormat='{{pod}}',
+        ),
+    ],
+)
+
+bucket_processor_buckets = TimeSeries(
+    title="Bucket Processor Lifecycle Buckets",
+    description="Number of lifecycle-enabled buckets processed by "
+        "each bucket processor during the current scan.",
+    dataSource="${DS_PROMETHEUS}",
+    lineInterpolation="smooth",
+    targets=[
+        Target(
+            expr=Metrics.BUCKET_PROCESSOR_BUCKETS_COUNT(),
+            legendFormat='{{pod}}',
+        ),
+    ],
+)
+
 active_indexing_jobs = TimeSeries(
     title="Active Indexing jobs",
     dataSource="${DS_PROMETHEUS}",
@@ -897,6 +981,8 @@ def color_override(name, color):
             layout.row([s3_delete_object_ops, s3_delete_mpu_ops], height=8),
             RowPanel(title="Lifecycle Conductor"),
             layout.row([lifecycle_scans, trigger_latency], height=7),
+            layout.row([lifecycle_full_scan_elapsed, lifecycle_scan_count], height=7),
+            layout.row([bucket_processor_scan_progress, bucket_processor_buckets], height=7),
             layout.row([lifecycle_scan_rate, active_indexing_jobs, legacy_tasks], height=7),
         ]),
     )