Skip to content

Commit f3b51ef

Browse files
committed
add notificaiton alert solved procedure
1 parent f1a4624 commit f3b51ef

File tree

5 files changed

+158
-1
lines changed

5 files changed

+158
-1
lines changed

configs.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ controller_procedures:
3232
schedule: "*/5 * * * *"
3333
params:
3434
time_tolerance: 10
35+
notifications_alert_solved:
36+
schedule: "*/5 * * * *"
3537

3638
executor_concurrency: 5
3739
executor_sleep: 5
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
from typing import Callable, Coroutine
22

3-
from . import monitors_stuck
3+
from . import monitors_stuck, notifications_alert_solved
44

55
procedures: dict[str, Callable[..., Coroutine[None, None, None]]] = {
66
"monitors_stuck": monitors_stuck.monitors_stuck,
7+
"notifications_alert_solved": notifications_alert_solved.notifications_alert_solved,
78
}
89

910
__all__ = ["procedures"]
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
"""
2+
Search for active notifications linked to alerts that have already been solved and closes the
3+
identified notifications.
4+
"""
5+
6+
import logging
7+
8+
import databases
9+
from models import Notification
10+
11+
from .constants import SQL_FILES_PATH
12+
13+
_logger = logging.getLogger("procedures.notifications_alert_solved")
14+
15+
16+
async def notifications_alert_solved() -> None:
17+
with open(SQL_FILES_PATH / "notification_alert_solved.sql") as file:
18+
query = file.read()
19+
20+
result = await databases.query_application(query)
21+
22+
if result is None:
23+
_logger.error("Error with query result")
24+
return
25+
26+
for notification_info in result:
27+
notification = await Notification.get_by_id(notification_info["id"])
28+
if notification is None:
29+
_logger.error(f"Notification with id '{notification_info['id']}' not found")
30+
continue
31+
32+
await notification.close()
33+
_logger.warning(f"{notification} closed")
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
with active_notifications as (
2+
select
3+
id,
4+
alert_id as notification_alert_id,
5+
status as notification_status
6+
from "Notifications"
7+
where
8+
status = 'active'
9+
)
10+
select active_notifications.id as id
11+
from active_notifications
12+
inner join "Alerts" as alerts
13+
on active_notifications.notification_alert_id = alerts.id
14+
where
15+
alerts.status = 'solved' and
16+
alerts.solved_at < current_timestamp - interval '5 minutes';
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
from datetime import datetime, timedelta
2+
from unittest.mock import AsyncMock
3+
4+
import pytest
5+
6+
import components.controller.procedures.notifications_alert_solved as notifications_alert_solved
7+
from models import Alert, AlertStatus, Monitor, Notification, NotificationStatus
8+
from tests.test_utils import assert_message_in_log, assert_message_not_in_log
9+
from utils.time import now
10+
11+
pytestmark = pytest.mark.asyncio(loop_scope="session")
12+
13+
14+
def get_time(reference: str) -> datetime | None:
15+
values = {
16+
"now": now(),
17+
"ten_seconds_ago": now() - timedelta(seconds=11),
18+
"five_minutes_ago": now() - timedelta(seconds=301),
19+
}
20+
return values.get(reference)
21+
22+
23+
@pytest.mark.parametrize("alert_status", [AlertStatus.active, AlertStatus.solved])
24+
@pytest.mark.parametrize(
25+
"notification_status", [NotificationStatus.active, NotificationStatus.closed]
26+
)
27+
@pytest.mark.parametrize("alert_solved_at", [None, "now", "five_minutes_ago"])
28+
async def test_notification_alert_solved(
29+
caplog, sample_monitor: Monitor, monkeypatch, alert_status, notification_status, alert_solved_at
30+
):
31+
"""'_notification_alert_solved' should close notifications that are active but the alert is
32+
already solved"""
33+
alert = await Alert.create(
34+
monitor_id=sample_monitor.id,
35+
status=alert_status,
36+
solved_at=get_time(alert_solved_at), # type:ignore[assignment]
37+
)
38+
notification = await Notification.create(
39+
monitor_id=sample_monitor.id, alert_id=alert.id, target="", status=notification_status
40+
)
41+
42+
if alert_status == AlertStatus.active:
43+
triggered = False
44+
elif notification_status == NotificationStatus.closed:
45+
triggered = False
46+
else:
47+
triggered = alert_solved_at == "five_minutes_ago"
48+
49+
await notifications_alert_solved.notifications_alert_solved()
50+
51+
await notification.refresh()
52+
if triggered:
53+
assert notification.status == NotificationStatus.closed
54+
assert notification.closed_at > now() - timedelta(seconds=1)
55+
assert_message_in_log(caplog, f"{notification} closed")
56+
else:
57+
assert notification.status == notification_status
58+
assert notification.closed_at is None
59+
assert_message_not_in_log(caplog, f"{notification} closed")
60+
61+
62+
async def test_notifications_alert_solved_query_result_none(caplog, monkeypatch):
63+
"""'notifications_alert_solved' should log an error if the query result is None"""
64+
monkeypatch.setattr(
65+
notifications_alert_solved.databases, "query_application", AsyncMock(return_value=None)
66+
)
67+
68+
await notifications_alert_solved.notifications_alert_solved()
69+
70+
assert_message_in_log(caplog, "Error with query result")
71+
72+
73+
async def test_notifications_alert_solved_monitor_not_found(caplog, monkeypatch):
74+
"""'notifications_alert_solved' should log an error if the notification is not found"""
75+
monkeypatch.setattr(
76+
notifications_alert_solved.databases,
77+
"query_application",
78+
AsyncMock(return_value=[{"id": 99999999}]),
79+
)
80+
81+
await notifications_alert_solved.notifications_alert_solved()
82+
83+
assert_message_in_log(caplog, "Notification with id '99999999' not found")
84+
85+
86+
async def test_notifications_alert_solved_monitor_not_found_2_results(
87+
caplog, monkeypatch, sample_monitor: Monitor
88+
):
89+
"""'notifications_alert_solved' should log an error if one notification is not found but should
90+
continue with the other notifications"""
91+
alert = await Alert.create(monitor_id=sample_monitor.id, status=AlertStatus.active)
92+
notification = await Notification.create(
93+
monitor_id=sample_monitor.id, alert_id=alert.id, target="", status=NotificationStatus.active
94+
)
95+
96+
monkeypatch.setattr(
97+
notifications_alert_solved.databases,
98+
"query_application",
99+
AsyncMock(return_value=[{"id": 99999999}, {"id": notification.id}]),
100+
)
101+
102+
await notifications_alert_solved.notifications_alert_solved()
103+
104+
assert_message_in_log(caplog, "Notification with id '99999999' not found")
105+
assert_message_in_log(caplog, f"{notification} closed")

0 commit comments

Comments
 (0)