Skip to content

Commit 5b4a99a

Browse files
vincent-olivert-rieravincent-olivert-riera
authored
[IMPROVEMENT] Notifiers: Add support for PargerDuty #686 
2 parents a00a7d8 + ee43a77 commit 5b4a99a

9 files changed

Lines changed: 275 additions & 10 deletions

File tree

‎promgen/mixins.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
from django.contrib.contenttypes.models import ContentType
88
from django.shortcuts import get_object_or_404
99
from django.views.generic.base import ContextMixin
10+
from django.views.generic.edit import FormView
1011

11-
from promgen import models
12+
from promgen import forms, models, notification
1213

1314

1415
class ContentTypeMixin:
@@ -78,3 +79,20 @@ def get_context_data(self, **kwargs):
7879
models.Service, id=self.kwargs["pk"]
7980
)
8081
return context
82+
83+
84+
class NotifierFormMixin(FormView):
85+
model = models.Sender
86+
form_class = forms.SenderForm
87+
88+
def post(self, request, *args, **kwargs):
89+
notifier_form = notification.load(request.POST["sender"]).form(request.POST)
90+
if notifier_form.is_valid():
91+
data = request.POST.copy()
92+
data.update(notifier_form.cleaned_data)
93+
sender_form = self.form_class(data)
94+
if sender_form.is_valid():
95+
return self.form_valid(sender_form)
96+
else:
97+
return self.form_invalid(sender_form)
98+
return self.form_invalid(notifier_form)

‎promgen/notification/pagerduty.py‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# Copyright (c) 2026 LINE Corporation
2+
# These sources are released under the terms of the MIT license: see LICENSE
3+
4+
"""
5+
Simple integration with PagerDuty Events V2 API.
6+
7+
Accepts alert json from Alert Manager and then POSTs individual alerts to PagerDuty.
8+
"""
9+
10+
import logging
11+
12+
from django import forms
13+
14+
from promgen import settings, util, validators
15+
from promgen.notification import NotificationBase
16+
from promgen.shortcuts import resolve_domain
17+
18+
logger = logging.getLogger(__name__)
19+
20+
21+
def _choices():
22+
if not getattr(settings, "PAGERDUTY_URLS", None):
23+
yield "PagerDuty", "PagerDuty"
24+
else:
25+
for k, v in settings.PAGERDUTY_URLS.items():
26+
yield (k, k)
27+
28+
29+
class FormPagerDuty(forms.Form):
30+
domain = forms.ChoiceField(
31+
required=True,
32+
label="PagerDuty Domain",
33+
choices=_choices(),
34+
)
35+
integration_key = forms.CharField(
36+
required=True,
37+
label="Integration Key",
38+
help_text="The Integration Key (a.k.a. Routing Key) provided by PagerDuty.",
39+
validators=[validators.integration_key],
40+
widget=forms.PasswordInput(),
41+
)
42+
alias = forms.CharField(
43+
label="Alias",
44+
help_text="Description for identifying this PagerDuty integration.",
45+
required=True,
46+
)
47+
48+
def clean(self):
49+
cleaned_data = super().clean()
50+
# The Sender model expects a single 'value' field. Therefore, we create it by combining
51+
# the domain and integration_key into a single string separated by a colon.
52+
cleaned_data["value"] = str.format(
53+
"{}:{}", cleaned_data.get("domain"), cleaned_data.get("integration_key")
54+
)
55+
return cleaned_data
56+
57+
58+
class NotificationPagerDuty(NotificationBase):
59+
"""
60+
Trigger an alert event to PagerDuty.
61+
"""
62+
63+
form = FormPagerDuty
64+
65+
def _send(self, target, data):
66+
domain = target.split(":")[0]
67+
integration_key = target.split(":")[1]
68+
69+
request_body = NotificationPagerDuty.build_request_body(data, integration_key)
70+
71+
# Send event to PagerDuty Events V2 API
72+
# By default we send to the global endpoint unless overridden
73+
url = "https://events.pagerduty.com/v2/enqueue"
74+
if getattr(settings, "PAGERDUTY_URLS", None):
75+
url = settings.PAGERDUTY_URLS.get(domain)
76+
77+
util.post(url, json=request_body).raise_for_status()
78+
79+
@staticmethod
80+
def json_to_string(data):
81+
return "".join(f"\n - {k} = {v}" for k, v in data.items())
82+
83+
@staticmethod
84+
def build_request_body(data, integration_key):
85+
# Initialize request body
86+
request_body = {
87+
"routing_key": integration_key,
88+
"client": "Promgen",
89+
"client_url": resolve_domain("home"),
90+
}
91+
92+
# Set event action
93+
if data["status"] == "resolved":
94+
request_body["event_action"] = "resolve"
95+
else:
96+
request_body["event_action"] = "trigger"
97+
98+
# Set dedup key
99+
# We added "Promgen/" prefix to avoid collision with other systems
100+
request_body["dedup_key"] = str.format("Promgen/{}", util.fingerprint(data))
101+
102+
# If we are triggering an alert, set extra fields, otherwise just send the resolve event
103+
if request_body["event_action"] == "resolve":
104+
return request_body
105+
106+
# Set link to Promgen alert
107+
if "externalURL" in data:
108+
request_body["links"] = [
109+
{
110+
"href": data["externalURL"],
111+
"text": "View alert in Promgen",
112+
}
113+
]
114+
115+
# Initialize payload
116+
request_body.setdefault("payload", {})
117+
request_body["payload"]["source"] = "Promgen"
118+
119+
# Set severity
120+
PAGERDUTY_ACCEPTED_SEVERITIES = {"info", "warning", "error", "critical"}
121+
severity = data["commonLabels"].get("severity")
122+
if severity in PAGERDUTY_ACCEPTED_SEVERITIES:
123+
request_body["payload"]["severity"] = severity
124+
elif severity and severity in settings.PAGERDUTY_SEVERITY_MAP:
125+
request_body["payload"]["severity"] = settings.PAGERDUTY_SEVERITY_MAP[severity]
126+
else:
127+
request_body["payload"]["severity"] = "error"
128+
129+
# Set summary with a max length of 1024 characters
130+
# https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event
131+
if "summary" in data["commonAnnotations"]:
132+
request_body["payload"]["summary"] = data["commonAnnotations"]["summary"][:1024]
133+
else:
134+
request_body["payload"]["summary"] = data["commonLabels"]["alertname"][:1024]
135+
136+
# Set custom details
137+
custom_details = {}
138+
custom_details["firing"] = "Labels:{}\nAnnotations:{} \n".format(
139+
NotificationPagerDuty.json_to_string(data["commonLabels"]),
140+
NotificationPagerDuty.json_to_string(data["commonAnnotations"]),
141+
)
142+
request_body["payload"]["custom_details"] = custom_details
143+
144+
return request_body

‎promgen/settings.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,15 @@
225225
"guardian.backends.ObjectPermissionBackend",
226226
)
227227

228+
# PagerDuty only accepts these severity levels: info, warning, error, critical.
229+
# However, Promgen user can define their own severity levels, so we provide
230+
# a mapping here to convert user-defined severity levels to PagerDuty accepted values.
231+
PAGERDUTY_SEVERITY_MAP = {
232+
"debug": "info",
233+
"minor": "warning",
234+
"major": "error",
235+
}
236+
228237
# Load overrides from PROMGEN to replace Django settings
229238
for k, v in PROMGEN.pop("django", {}).items():
230239
globals()[k] = v

‎promgen/tests/notification/pagerduty.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"routing_key" : "integration_key_test",
3+
"client" : "Promgen",
4+
"client_url" : "http://promgen.example.com/",
5+
"event_action" : "trigger",
6+
"dedup_key" : "Promgen/0bb9bd7951ad3ef749586f77f98177427df4af00",
7+
"links" : [ {
8+
"href" : "http://promgen.example.com/alert/1",
9+
"text" : "View alert in Promgen"
10+
} ],
11+
"payload" : {
12+
"source" : "Promgen",
13+
"severity" : "error",
14+
"summary" : "test-alert",
15+
"custom_details" : {
16+
"firing" : "Labels:\n - alertname = test-alert\n - project = test-project\n - service = test-service\n - severity = major\nAnnotations:\n - grafana = http://grafana.example.com/dashboard/db/overview?var-service=test-service&var-project=test-project\n - project = http://promgen.example.com/project/1\n - service = http://promgen.example.com/service/1 \n"
17+
}
18+
}
19+
}

‎promgen/tests/notification/test_pagerduty.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright (c) 2017 LINE Corporation
2+
# These sources are released under the terms of the MIT license: see LICENSE
3+
4+
from unittest import mock
5+
6+
from django.contrib.auth.models import Permission
7+
from django.test import override_settings
8+
9+
from promgen import models, rest, tests
10+
from promgen.notification.pagerduty import NotificationPagerDuty
11+
12+
13+
class PagerDutyTest(tests.PromgenTest):
14+
def setUp(self):
15+
one = models.Project.objects.get(pk=1)
16+
two = models.Service.objects.get(pk=1)
17+
18+
self.senderA = NotificationPagerDuty.create(
19+
obj=one, value="PagerDuty:integration_key_test", owner_id=1
20+
)
21+
self.senderB = NotificationPagerDuty.create(
22+
obj=two, value="PagerDuty Proxy Server:integration_key_test", owner_id=1
23+
)
24+
25+
self.user = self.force_login(username="demo")
26+
permission = Permission.objects.get(codename="process_alert")
27+
self.user.user_permissions.add(permission)
28+
29+
@override_settings(PROMGEN=tests.SETTINGS)
30+
@override_settings(CELERY_TASK_ALWAYS_EAGER=True)
31+
@override_settings(CELERY_TASK_EAGER_PROPAGATES=True)
32+
@mock.patch("promgen.util.post")
33+
def test_pagerduty(self, mock_post):
34+
response = self.fireAlert()
35+
self.assertRoute(response, rest.AlertReceiver, 202)
36+
self.assertCount(models.AlertError, 0, "No failed alerts")
37+
38+
self.assertCount(models.Alert, 1, "Alert should be queued")
39+
self.assertEqual(mock_post.call_count, 2, "Two alerts should be sent")
40+
41+
# Our sample is the same as the original, with some annotations added
42+
_SAMPLE = tests.Data("notification", "pagerduty.json").json()
43+
# External URL is depended on test order
44+
_SAMPLE["links"][0]["href"] = mock.ANY
45+
46+
mock_post.assert_has_calls(
47+
[
48+
mock.call("https://events.pagerduty.com/v2/enqueue", json=_SAMPLE),
49+
mock.call("https://events.pagerduty.com/v2/enqueue", json=_SAMPLE),
50+
],
51+
any_order=True,
52+
)

‎promgen/util.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# These sources are released under the terms of the MIT license: see LICENSE
33

44
import argparse
5+
import hashlib
56
import math
67
from urllib.parse import urlsplit
78

@@ -173,6 +174,15 @@ def categorize_error(e: Exception) -> str:
173174
return "other_error"
174175

175176

177+
def fingerprint(body):
178+
buff = ""
179+
for k, v in sorted(body.get("groupLabels", {}).items()):
180+
buff += k
181+
buff += v
182+
183+
return hashlib.sha1(buff.encode("utf8")).hexdigest()
184+
185+
176186
# Comment wrappers to get the docstrings from the upstream functions
177187
get.__doc__ = requests.get.__doc__
178188
post.__doc__ = requests.post.__doc__

‎promgen/validators.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@
5353
schemes=["http", "https"],
5454
)
5555

56+
# https://support.pagerduty.com/main/docs/api-access-keys
57+
integration_key = RegexValidator(
58+
regex=r"^[a-zA-Z0-9]{32}$",
59+
message="Integration Key must be 32 alphanumeric characters.",
60+
)
61+
5662

5763
def datetime(value):
5864
try:

‎promgen/views.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -968,10 +968,8 @@ def form_valid(self, form):
968968
return HttpResponseRedirect(project.get_absolute_url() + "#hosts")
969969

970970

971-
class ProjectNotifierRegister(LoginRequiredMixin, FormView, mixins.ProjectMixin):
972-
model = models.Sender
971+
class ProjectNotifierRegister(LoginRequiredMixin, mixins.NotifierFormMixin, mixins.ProjectMixin):
973972
template_name = "promgen/notifier_form.html"
974-
form_class = forms.SenderForm
975973

976974
def form_valid(self, form):
977975
project = get_object_or_404(models.Project, id=self.kwargs["pk"])
@@ -983,11 +981,13 @@ def form_valid(self, form):
983981
signals.check_user_subscription(models.Sender, sender, created, self.request)
984982
return HttpResponseRedirect(project.get_absolute_url() + "#notifiers")
985983

984+
def form_invalid(self, form):
985+
messages.error(self.request, "Error creating notifier: %s" % form.errors.as_text())
986+
return super().form_invalid(form)
986987

987-
class ServiceNotifierRegister(LoginRequiredMixin, FormView, mixins.ServiceMixin):
988-
model = models.Sender
988+
989+
class ServiceNotifierRegister(LoginRequiredMixin, mixins.NotifierFormMixin, mixins.ServiceMixin):
989990
template_name = "promgen/notifier_form.html"
990-
form_class = forms.SenderForm
991991

992992
def form_valid(self, form):
993993
service = get_object_or_404(models.Service, id=self.kwargs["pk"])
@@ -999,6 +999,10 @@ def form_valid(self, form):
999999
signals.check_user_subscription(models.Sender, sender, created, self.request)
10001000
return HttpResponseRedirect(service.get_absolute_url() + "#notifiers")
10011001

1002+
def form_invalid(self, form):
1003+
messages.error(self.request, "Error creating notifier: %s" % form.errors.as_text())
1004+
return super().form_invalid(form)
1005+
10021006

10031007
class SiteDetail(LoginRequiredMixin, TemplateView):
10041008
template_name = "promgen/site_detail.html"
@@ -1011,9 +1015,7 @@ def get_context_data(self, **kwargs):
10111015
return context
10121016

10131017

1014-
class Profile(LoginRequiredMixin, FormView):
1015-
form_class = forms.SenderForm
1016-
model = models.Sender
1018+
class Profile(LoginRequiredMixin, mixins.NotifierFormMixin):
10171019
template_name = "promgen/profile.html"
10181020

10191021
def get_context_data(self, **kwargs):
@@ -1033,6 +1035,10 @@ def form_valid(self, form):
10331035
)
10341036
return redirect("profile")
10351037

1038+
def form_invalid(self, form):
1039+
messages.error(self.request, "Error creating notifier: %s" % form.errors.as_text())
1040+
return super().form_invalid(form)
1041+
10361042

10371043
class HostRegister(LoginRequiredMixin, FormView):
10381044
model = models.Host

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ promgen = "promgen.discovery.default:DiscoveryPromgen"
5858
[project.entry-points."promgen.notification"]
5959
email = "promgen.notification.email:NotificationEmail"
6060
linenotify = "promgen.notification.linenotify:NotificationLineNotify"
61+
pagerduty = "promgen.notification.pagerduty:NotificationPagerDuty"
6162
slack = "promgen.notification.slack:NotificationSlack"
6263
user = "promgen.notification.user:NotificationUser"
6364
webhook = "promgen.notification.webhook:NotificationWebhook"

0 commit comments

Comments
 (0)