forked from opendatahub-io/opendatahub-tests
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmonitoring.py
More file actions
88 lines (71 loc) · 2.65 KB
/
monitoring.py
File metadata and controls
88 lines (71 loc) · 2.65 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
from typing import Any, Callable
from ocp_resources.prometheus import Prometheus
from simple_logger.logger import get_logger
from timeout_sampler import TimeoutExpiredError, TimeoutSampler
LOGGER = get_logger(name=__name__)
def get_metrics_value(prometheus: Prometheus, metrics_query: str) -> Any:
"""
Get metrics value from prometheus
Args:
prometheus (Prometheus): Prometheus object
metrics_query (str): Metrics query string
Returns:
Any: Metrics value
"""
metric_results = prometheus.query_sampler(query=metrics_query)
if metric_values_list := [value for metric_val in metric_results for value in metric_val.get("value")]:
return metric_values_list[1]
def get_metric_label(
prometheus: Prometheus,
metrics_query: str,
label_name: str,
) -> Any:
"""
Get the value of a specific label from the first matching metric.
Args:
prometheus (Prometheus): Prometheus object
metrics_query (str): Metrics query string
label_name (str): Label to retrieve
Returns:
Any: Value of the requested label, or None if not found
"""
metric_results = prometheus.query_sampler(query=metrics_query)
LOGGER.info(f"Fields: {metric_results}")
if metric_results:
# Assume we care about the first result
return metric_results[0]["metric"].get(label_name)
return None
def validate_metrics_field(
prometheus: Prometheus,
metrics_query: str,
expected_value: Any,
field_getter: Callable[..., Any] = get_metrics_value,
timeout: int = 60 * 4,
) -> None:
"""
Validate any metric field or label using a custom getter function.
Defaults to checking the metric's value if no getter is provided.
Args:
prometheus (Prometheus): Prometheus object
metrics_query (str): Metrics query string
expected_value (Any): Expected value
field_getter (Callable): Function to extract the desired field/label/value
timeout (int): Timeout in seconds
Raises:
TimeoutExpiredError: If expected value isn't met within the timeout
"""
try:
for sample in TimeoutSampler(
wait_timeout=timeout,
sleep=15,
func=field_getter,
prometheus=prometheus,
metrics_query=metrics_query,
):
if sample == expected_value:
LOGGER.info("Metric field matches the expected value!")
return
LOGGER.info(f"Current value: {sample}, waiting for: {expected_value}")
except TimeoutExpiredError:
LOGGER.error(f"Timed out. Last value: {sample}, expected: {expected_value}")
raise