Skip to content

Alertmanager 0.1 and restricting to tracked projects #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Alertmanager 0.1 and restricting to tracked projects #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
db91eff
plugins/prometheus: Add rudimentary support for alertmanager 0.1
superdump May 9, 2016
3053e15
plugins/prometheus: Add tracking support
superdump Jun 22, 2016
26e8008
prometheus.json: Fix missing known_projects configuration
superdump Jun 22, 2016
e02db05
plugins/prometheus: Update documentation
superdump Jun 22, 2016
69bcdc1
prometheus.json: Add default project
superdump Jun 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 119 additions & 7 deletions plugins/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,20 @@


class PrometheusPlugin(Plugin):
"""Plugin for interacting with Prometheus.
""" Plugin for receiving Prometheus alerts.
prometheus show projects : Display which projects this bot recognises.
prometheus show track|tracking : Display which projects this bot is tracking.
prometheus track project1 project2 ... : Track Prometheus notifications for the named projects.
prometheus stop track|tracking : Stop tracking Prometheus notifications.
prometheus add projectName : Start tracking projectName.
prometheus remove projectName : Stop tracking projectName.
"""
name = "prometheus"

#Webhooks:
# /neb/prometheus

TRACKING = ["track", "tracking"]
TYPE_TRACK = "org.matrix.neb.plugin.prometheus.projects.tracking"


Expand All @@ -35,6 +43,101 @@ def __init__(self, *args, **kwargs):
self.consumer.daemon = True
self.consumer.start()


def cmd_show(self, event, action):
"""Show information on projects or projects being tracked.
Show which projects are being tracked. 'prometheus show tracking'
Show which proejcts are recognised so they could be tracked. 'prometheus show projects'
"""
if action in self.TRACKING:
return self._get_tracking(event["room_id"])
elif action == "projects":
projects = self.store.get("known_projects")
return "Available projects: %s" % json.dumps(projects)
else:
return "Invalid arg '%s'.\n %s" % (action, self.cmd_show.__doc__)

@admin_only
def cmd_track(self, event, *args):
"""Track projects. 'prometheus track Foo "bar with spaces"'"""
if len(args) == 0:
return self._get_tracking(event["room_id"])

for project in args:
if not project in self.store.get("known_projects"):
return "Unknown project name: %s." % project

self._send_track_event(event["room_id"], args)

return "Prometheus notifications for projects %s will be displayed when they fail." % (args)

@admin_only
def cmd_add(self, event, project):
"""Add a project for tracking. 'prometheus add projectName'"""
if project not in self.store.get("known_projects"):
return "Unknown project name: %s." % project

try:
room_projects = self.rooms.get_content(
event["room_id"],
PrometheusPlugin.TYPE_TRACK)["projects"]
Copy link

@mweinelt mweinelt Jun 22, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe put the closing bracket ) into the newline, to increase readability.

Copy link
Author

@superdump superdump Jun 27, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. This was a copy and paste from the jenkins plugin with only indentation changes. I thought it was difficult to see the closing ) too.

except KeyError:
room_projects = []

if project in room_projects:
return "%s is already being tracked." % project

room_projects.append(project)
self._send_track_event(event["room_id"], room_projects)

return "Added %s. Prometheus notifications for projects %s will be displayed when they fail." % (project, room_projects)

@admin_only
def cmd_remove(self, event, project):
"""Remove a project from tracking. 'prometheus remove projectName'"""
try:
room_projects = self.rooms.get_content(
event["room_id"],
PrometheusPlugin.TYPE_TRACK)["projects"]
except KeyError:
room_projects = []

if project not in room_projects:
return "Cannot remove %s : It isn't being tracked." % project

room_projects.remove(project)
self._send_track_event(event["room_id"], room_projects)

return "Removed %s. Prometheus notifications for projects %s will be displayed when they fail." % (project, room_projects)

@admin_only
def cmd_stop(self, event, action):
"""Stop tracking projects. 'prometheus stop tracking'"""
if action in self.TRACKING:
self._send_track_event(event["room_id"], [])
return "Stopped tracking projects."
else:
return "Invalid arg '%s'.\n %s" % (action, self.cmd_stop.__doc__)

def _get_tracking(self, room_id):
try:
return ("Currently tracking %s" %
json.dumps(self.rooms.get_content(
room_id, PrometheusPlugin.TYPE_TRACK)["projects"]
)
)
except KeyError:
return "Not tracking any projects currently."

def _send_track_event(self, room_id, project_names):
self.matrix.send_state_event(
room_id,
self.TYPE_TRACK,
{
"projects": project_names
}
)

def on_event(self, event, event_type):
self.rooms.update(event)

Expand All @@ -48,12 +151,21 @@ def get_webhook_key(self):
def on_receive_webhook(self, url, data, ip, headers):
json_data = json.loads(data)
log.info("recv %s", json_data)
template = Template(self.store.get("message_template"))
for alert in json_data.get("alert", []):
for room_id in self.rooms.get_room_ids():
log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert)
queue.put((self.queue_counter, room_id, template.render(alert)))
self.queue_counter += 1
def process_alerts(alerts, template):
for alert in alerts:
for room_id in self.rooms.get_room_ids():
try:
if (len(self.rooms.get_content(
room_id, PrometheusPlugin.TYPE_TRACK)["projects"])):
Copy link
Member

@kegsay kegsay Jun 22, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your check here is merely checking for a non-zero length array, because you haven't yet implemented the known_projects thing. Please can you put some comments explaining:

  • Why you're checking for a non-zero length array
  • What needs to be done in order for it to be actually checking the projectName.

log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert)
queue.put((self.queue_counter, room_id, template.render(alert)))
self.queue_counter += 1
except KeyError:
pass
# try version 1 format
process_alerts(json_data.get("alert", []), Template(self.store.get("v1_message_template")))
# try version 2+ format
process_alerts(json_data.get("alerts", []), Template(self.store.get("v2_message_template")))
Copy link
Member

@kegsay kegsay Jun 22, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your comments say "try" - does this mean this is an either/or situation? If so, can we please have the control flow indicate this (e.g. via an if else) rather than blindly processing both types.

If it is possible for both types to co-exist in the same webhook, then can you tweak the wording so it's less confusing, thanks!

Copy link
Author

@superdump superdump Jun 27, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find the code as it is to be much simpler and more readable. It should be an either or situation, but process_alerts() does nothing in the case that there are no alerts. The alternative is code that looks like this:

        alerts = json_data.get("alert", [])
        if len(alerts) > 0:
            process_alerts(alerts, Template(self.store.get("v1_message_template")))
        else:
            alerts = json_data.get("alerts", [])
            if len(alerts) > 0:
                process_alerts(, Template(self.store.get("v2_message_template")))

Which to me feels unnecessarily verbose. I will make the change to that if you want me to.



class MessageConsumer(Thread):
Expand Down
6 changes: 4 additions & 2 deletions prometheus.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
{
"message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}"
}
"known_projects": ["prometheus"],
"v1_message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}",
"v2_message_template": "[ALERT] {{labels.alertname}} : {{annotations.summary}}\n{{labels.job}} on {{labels.instance}}\n{{status}}\n{{annotations.description}}\n{{generatorURL}}\nStarted: {{startsAt}} ; Ended: {{endsAt}}"
}