Add support to auto resolve jira issue when alert is resolved. (#117)

* Add initial code to support auto resolve feature

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* use constant

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Resolve alert using transition

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Update configuration tests

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Update example configuration

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Update README

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* move valid config check to receivers section

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Use different configuration option for simplicity

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* remove old config options

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* add docstring and update readme

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* reuse AutoResolve struct in tests

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* add docstring for AutoResolve

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* Apply suggestions from code review

Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* keep the reasoning why defaults is not auto-resolve

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* fix lint errors

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* keep newline in configuration yaml

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

* fix readme

Signed-off-by: Bhushan Thakur <bhthakur@redhat.com>

Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>

Author: bhushanthakur93

README.md

@@ -7,7 +7,7 @@
 
 ## Overview
 
-JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#<route>) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it.  If no human interaction is necessary then it should probably not alert in the first place.
+JIRAlert implements Alertmanager's webhook HTTP API and connects to one or more JIRA instances to create highly configurable JIRA issues. One issue is created per distinct group key — as defined by the [`group_by`](https://prometheus.io/docs/alerting/configuration/#<route>) parameter of Alertmanager's `route` configuration section — but not closed when the alert is resolved. The expectation is that a human will look at the issue, take any necessary action, then close it.  If no human interaction is necessary then it should probably not alert in the first place. This behavior however can be modified by setting `auto_resolve` section, which will resolve the jira issue with required state.
 
 If a corresponding JIRA issue already exists but is resolved, it is reopened. A JIRA transition must exist between the resolved state and the reopened state — as defined by `reopen_state` — or reopening will fail. Optionally a "won't fix" resolution — defined by `wont_fix_resolution` — may be defined: a JIRA issue with this resolution will not be reopened by JIRAlert.
 

examples/jiralert.yml

@@ -47,6 +47,10 @@ receivers:
       customfield_10002: {"value": "red"}
       # MultiSelect
       customfield_10003: [{"value": "red"}, {"value": "blue"}, {"value": "green"}]
+    #
+    # Automatically resolve jira issues when alert is resolved. Optional. If declared, ensure state is not an empty string.
+    auto_resolve:
+      state: 'Done' 
 
 # File containing template definitions. Required.
 template: jiralert.tmpl

pkg/alertmanager/alertmanager.go

@@ -27,6 +27,9 @@ const (
 
 	// AlertFiring is the status value for a firing alert.
 	AlertFiring = "firing"
+
+	// AlertResolved is the status value for a resolved alert.
+	AlertResolved = "resolved"
 )
 
 // Pair is a key/value string pair.

pkg/config/config.go

@@ -88,6 +88,11 @@ func resolveFilepaths(baseDir string, cfg *Config, logger log.Logger) {
 	cfg.Template = join(cfg.Template)
 }
 
+// AutoResolve is the struct used for defining jira resolution state when alert is resolved.
+type AutoResolve struct {
+	State string `yaml:"state" json:"state"`
+}
+
 // ReceiverConfig is the configuration for one receiver. It has a unique name and includes API access fields (url and
 // auth) and issue fields (required -- e.g. project, issue type -- and optional -- e.g. priority).
 type ReceiverConfig struct {
@@ -116,6 +121,9 @@ type ReceiverConfig struct {
 	// Label copy settings
 	AddGroupLabels bool `yaml:"add_group_labels" json:"add_group_labels"`
 
+	// Flag to auto-resolve opened issue when the alert is resolved.
+	AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"`
+
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline" json:"-"`
 }
@@ -171,6 +179,12 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("bad auth config in defaults section: user/password and PAT authentication are mutually exclusive")
 	}
 
+	if c.Defaults.AutoResolve != nil {
+		if c.Defaults.AutoResolve.State == "" {
+			return fmt.Errorf("bad config in defaults section: state cannot be empty")
+		}
+	}
+
 	for _, rc := range c.Receivers {
 		if rc.Name == "" {
 			return fmt.Errorf("missing name for receiver %+v", rc)
@@ -251,6 +265,14 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		if rc.WontFixResolution == "" && c.Defaults.WontFixResolution != "" {
 			rc.WontFixResolution = c.Defaults.WontFixResolution
 		}
+		if rc.AutoResolve != nil {
+			if rc.AutoResolve.State == "" {
+				return fmt.Errorf("bad config in receiver %q, 'auto_resolve' was defined with empty 'state' field", rc.Name)
+			}
+		}
+		if rc.AutoResolve == nil && c.Defaults.AutoResolve != nil {
+			rc.AutoResolve = c.Defaults.AutoResolve
+		}
 		if len(c.Defaults.Fields) > 0 {
 			for key, value := range c.Defaults.Fields {
 				if _, ok := rc.Fields[key]; !ok {

pkg/config/config_test.go

@@ -111,6 +111,8 @@ type receiverTestConfig struct {
 	WontFixResolution string `yaml:"wont_fix_resolution,omitempty"`
 	AddGroupLabels    bool   `yaml:"add_group_labels,omitempty"`
 
+	AutoResolve *AutoResolve `yaml:"auto_resolve" json:"auto_resolve"`
+
 	// TODO(rporres): Add support for these.
 	// Fields            map[string]interface{} `yaml:"fields,omitempty"`
 	// Components        []string               `yaml:"components,omitempty"`
@@ -290,6 +292,7 @@ func TestAuthKeysOverrides(t *testing.T) {
 // No tests for auth keys here. They will be handled separately
 func TestReceiverOverrides(t *testing.T) {
 	fifteenHoursToDuration, err := ParseDuration("15h")
+	autoResolve := AutoResolve{State: "Done"}
 	require.NoError(t, err)
 
 	// We'll override one key at a time and check the value in the receiver.
@@ -308,8 +311,9 @@ func TestReceiverOverrides(t *testing.T) {
 		{"Description", "A nice description", "A nice description"},
 		{"WontFixResolution", "Won't Fix", "Won't Fix"},
 		{"AddGroupLabels", false, false},
+		{"AutoResolve", &AutoResolve{State: "Done"}, &autoResolve},
 	} {
-		optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels"}
+		optionalFields := []string{"Priority", "Description", "WontFixResolution", "AddGroupLabels", "AutoResolve"}
 		defaultsConfig := newReceiverTestConfig(mandatoryReceiverFields(), optionalFields)
 		receiverConfig := newReceiverTestConfig([]string{"Name"}, optionalFields)
 
@@ -361,6 +365,8 @@ func newReceiverTestConfig(mandatory []string, optional []string) *receiverTestC
 		var value reflect.Value
 		if name == "AddGroupLabels" {
 			value = reflect.ValueOf(true)
+		} else if name == "AutoResolve" {
+			value = reflect.ValueOf(&AutoResolve{State: "Done"})
 		} else {
 			value = reflect.ValueOf(name)
 		}
@@ -399,3 +405,41 @@ func mandatoryReceiverFields() []string {
 	return []string{"Name", "APIURL", "User", "Password", "Project",
 		"IssueType", "Summary", "ReopenState", "ReopenDuration"}
 }
+
+func TestAutoResolveConfigReceiver(t *testing.T) {
+	mandatory := mandatoryReceiverFields()
+	minimalReceiverTestConfig := &receiverTestConfig{
+		Name: "test",
+		AutoResolve: &AutoResolve{
+			State: "",
+		},
+	}
+
+	defaultsConfig := newReceiverTestConfig(mandatory, []string{})
+	config := testConfig{
+		Defaults:  defaultsConfig,
+		Receivers: []*receiverTestConfig{minimalReceiverTestConfig},
+		Template:  "jiralert.tmpl",
+	}
+
+	configErrorTestRunner(t, config, "bad config in receiver \"test\", 'auto_resolve' was defined with empty 'state' field")
+
+}
+
+func TestAutoResolveConfigDefault(t *testing.T) {
+	mandatory := mandatoryReceiverFields()
+	minimalReceiverTestConfig := newReceiverTestConfig([]string{"Name"}, []string{"AutoResolve"})
+
+	defaultsConfig := newReceiverTestConfig(mandatory, []string{})
+	defaultsConfig.AutoResolve = &AutoResolve{
+		State: "",
+	}
+	config := testConfig{
+		Defaults:  defaultsConfig,
+		Receivers: []*receiverTestConfig{minimalReceiverTestConfig},
+		Template:  "jiralert.tmpl",
+	}
+
+	configErrorTestRunner(t, config, "bad config in defaults section: state cannot be empty")
+
+}

pkg/notify/notify.go

@@ -101,6 +101,15 @@ func (r *Receiver) Notify(data *alertmanager.Data, hashJiraLabel bool) (bool, er
 		}
 
 		if len(data.Alerts.Firing()) == 0 {
+			if r.conf.AutoResolve != nil {
+				level.Debug(r.logger).Log("msg", "no firing alert; resolving issue", "key", issue.Key, "label", issueGroupLabel)
+				retry, err := r.resolveIssue(issue.Key)
+				if err != nil {
+					return retry, err
+				}
+				return false, nil
+			}
+
 			level.Debug(r.logger).Log("msg", "no firing alert; summary checked, nothing else to do.", "key", issue.Key, "label", issueGroupLabel)
 			return false, nil
 		}
@@ -343,24 +352,7 @@ func (r *Receiver) updateDescription(issueKey string, description string) (bool,
 }
 
 func (r *Receiver) reopen(issueKey string) (bool, error) {
-	transitions, resp, err := r.client.GetTransitions(issueKey)
-	if err != nil {
-		return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger)
-	}
-
-	for _, t := range transitions {
-		if t.Name == r.conf.ReopenState {
-			level.Debug(r.logger).Log("msg", "transition (reopen)", "key", issueKey, "transitionID", t.ID)
-			resp, err = r.client.DoTransition(issueKey, t.ID)
-			if err != nil {
-				return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger)
-			}
-
-			level.Debug(r.logger).Log("msg", "reopened", "key", issueKey)
-			return false, nil
-		}
-	}
-	return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
+	return r.doTransition(issueKey, r.conf.ReopenState)
 }
 
 func (r *Receiver) create(issue *jira.Issue) (bool, error) {
@@ -390,3 +382,29 @@ func handleJiraErrResponse(api string, resp *jira.Response, err error, logger lo
 	}
 	return false, errors.Wrapf(err, "JIRA request %s failed", api)
 }
+
+func (r *Receiver) resolveIssue(issueKey string) (bool, error) {
+	return r.doTransition(issueKey, r.conf.AutoResolve.State)
+}
+
+func (r *Receiver) doTransition(issueKey string, transitionState string) (bool, error) {
+	transitions, resp, err := r.client.GetTransitions(issueKey)
+	if err != nil {
+		return handleJiraErrResponse("Issue.GetTransitions", resp, err, r.logger)
+	}
+
+	for _, t := range transitions {
+		if t.Name == transitionState {
+			level.Debug(r.logger).Log("msg", fmt.Sprintf("transition %s", transitionState), "key", issueKey, "transitionID", t.ID)
+			resp, err = r.client.DoTransition(issueKey, t.ID)
+			if err != nil {
+				return handleJiraErrResponse("Issue.DoTransition", resp, err, r.logger)
+			}
+
+			level.Debug(r.logger).Log("msg", transitionState, "key", issueKey)
+			return false, nil
+		}
+	}
+	return false, errors.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
+
+}

pkg/notify/notify_test.go

@@ -46,7 +46,7 @@ type fakeJira struct {
 func newTestFakeJira() *fakeJira {
 	return &fakeJira{
 		issuesByKey:     map[string]*jira.Issue{},
-		transitionsByID: map[string]jira.Transition{},
+		transitionsByID: map[string]jira.Transition{"1234": {ID: "1234", Name: "Done"}},
 		keysByQuery:     map[string][]string{},
 	}
 }
@@ -174,6 +174,19 @@ func testReceiverConfig2() *config.ReceiverConfig {
 	}
 }
 
+func testReceiverConfigAutoResolve() *config.ReceiverConfig {
+	reopen := config.Duration(1 * time.Hour)
+	autoResolve := config.AutoResolve{State: "Done"}
+	return &config.ReceiverConfig{
+		Project:           "abc",
+		Summary:           `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}`,
+		ReopenDuration:    &reopen,
+		ReopenState:       "reopened",
+		WontFixResolution: "won't-fix",
+		AutoResolve:       &autoResolve,
+	}
+}
+
 func TestNotify_JIRAInteraction(t *testing.T) {
 	testNowTime := time.Now()
 
@@ -479,6 +492,49 @@ func TestNotify_JIRAInteraction(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:        "auto resolve alert",
+			inputConfig: testReceiverConfigAutoResolve(),
+			inputAlert: &alertmanager.Data{
+				Alerts: alertmanager.Alerts{
+					{Status: "resolved"},
+				},
+				Status:      alertmanager.AlertResolved,
+				GroupLabels: alertmanager.KV{"a": "b", "c": "d"},
+			},
+			initJira: func(t *testing.T) *fakeJira {
+				f := newTestFakeJira()
+				_, _, err := f.Create(&jira.Issue{
+					ID:  "1",
+					Key: "1",
+					Fields: &jira.IssueFields{
+						Project:     jira.Project{Key: testReceiverConfigAutoResolve().Project},
+						Labels:      []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"},
+						Unknowns:    tcontainer.MarshalMap{},
+						Summary:     "[FIRING:2] b d ",
+						Description: "1",
+					},
+				})
+				require.NoError(t, err)
+				return f
+			},
+			expectedJiraIssues: map[string]*jira.Issue{
+				"1": {
+					ID:  "1",
+					Key: "1",
+					Fields: &jira.IssueFields{
+						Project: jira.Project{Key: testReceiverConfigAutoResolve().Project},
+						Labels:  []string{"JIRALERT{819ba5ecba4ea5946a8d17d285cb23f3bb6862e08bb602ab08fd231cd8e1a83a1d095b0208a661787e9035f0541817634df5a994d1b5d4200d6c68a7663c97f5}"},
+						Status: &jira.Status{
+							StatusCategory: jira.StatusCategory{Key: "Done"},
+						},
+						Unknowns:    tcontainer.MarshalMap{},
+						Summary:     "[RESOLVED] b d ", // Title changed.
+						Description: "1",
+					},
+				},
+			},
+		},
 	} {
 		if ok := t.Run(tcase.name, func(t *testing.T) {
 			fakeJira := tcase.initJira(t)