fix: pageduty custom_details in payload

This change fixes the `custom_details` value in PagerDuty V2 API payloads.

Fixes #2477
Fixes #3218

Alertmanager PagerDuty configuration allows users to define `details` under `pagerduty_configs`:
```yaml
[ details: { <string>: <tmpl_string>, ... } | default = {
  firing:       '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
  resolved:     '{{ template "pagerduty.default.instances" .Alerts.Resolved }}'
  num_firing:   '{{ .Alerts.Firing | len }}'
  num_resolved: '{{ .Alerts.Resolved | len }}'
} ]
```

The internal Alertmanager configuration structure is defined as:
```go
// PagerdutyConfig configures notifications via PagerDuty.
type PagerdutyConfig struct {
...
	Details        map[string]string `yaml:"details,omitempty" json:"details,omitempty"`
}
```
And the PagerDuty payload is defined as:
```go
type pagerDutyPayload struct {
...
	CustomDetails map[string]string `json:"custom_details,omitempty"`
}
```

The current flow of configuration parsing and encoding is:
1. `pagerduty_configs[].details` are read from configuration as `map[string]string`
2. The `details` map values are parsed as text templates
3. The result is passed as `custom_details`, part of the JSON payload sent to PagerDuty API V2

Here is an example payload using `.Alerts.Firing`, which returns a list of currently firing alert objects in this group.
Documented here: https://prometheus.io/docs/alerting/latest/notifications/#data
This results in a payload like below:
```json
{
  "client": "Alertmanager",
...
  "custom_details": {
    "firing": "Labels:
 - alertname = Server_Down
...
Annotations:
 - summary = Server is down
...
"
    }
}
```
Using `map[string]string` in payloads cause the rendered YAML templates to be encoded as JSON strings.
This is undesirable in most cases as the end-user might expect a nested JSON object which is machine readable/parsable by it's fields.

This change:
- adds logic to unmarshall `details` values as YAML with fallback to string
- uses `map[string]interface{}` to encode `custom_details` to JSON
- requires gopkg.in/yaml.v3, see go-yaml/yaml#591
- does not change existing yaml.v2 dependency, see #3322

Signed-off-by: Siavash Safi <[email protected]>

Author: siavashs

go.mod

@@ -48,6 +48,7 @@ require (
 	golang.org/x/tools v0.23.0
 	gopkg.in/telebot.v3 v3.3.6
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -100,5 +101,4 @@ require (
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

notify/pagerduty/pagerduty.go

@@ -29,6 +29,7 @@ import (
 	"github.com/go-kit/log/level"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"gopkg.in/yaml.v3"
 
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
@@ -107,14 +108,14 @@ type pagerDutyImage struct {
 }
 
 type pagerDutyPayload struct {
-	Summary       string            `json:"summary"`
-	Source        string            `json:"source"`
-	Severity      string            `json:"severity"`
-	Timestamp     string            `json:"timestamp,omitempty"`
-	Class         string            `json:"class,omitempty"`
-	Component     string            `json:"component,omitempty"`
-	Group         string            `json:"group,omitempty"`
-	CustomDetails map[string]string `json:"custom_details,omitempty"`
+	Summary       string                 `json:"summary"`
+	Source        string                 `json:"source"`
+	Severity      string                 `json:"severity"`
+	Timestamp     string                 `json:"timestamp,omitempty"`
+	Class         string                 `json:"class,omitempty"`
+	Component     string                 `json:"component,omitempty"`
+	Group         string                 `json:"group,omitempty"`
+	CustomDetails map[string]interface{} `json:"custom_details,omitempty"`
 }
 
 func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
@@ -129,7 +130,7 @@ func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
 		if n.apiV1 != "" {
 			msg.Details = map[string]string{"error": truncatedMsg}
 		} else {
-			msg.Payload.CustomDetails = map[string]string{"error": truncatedMsg}
+			msg.Payload.CustomDetails = map[string]interface{}{"error": truncatedMsg}
 		}
 
 		warningMsg := fmt.Sprintf("Truncated Details because message of size %s exceeds limit %s", units.MetricBytes(buf.Len()).String(), units.MetricBytes(maxEventSize).String())
@@ -246,7 +247,7 @@ func (n *Notifier) notifyV2(
 			Summary:       summary,
 			Source:        tmpl(n.conf.Source),
 			Severity:      tmpl(n.conf.Severity),
-			CustomDetails: details,
+			CustomDetails: toCustomDetails(details),
 			Class:         tmpl(n.conf.Class),
 			Component:     tmpl(n.conf.Component),
 			Group:         tmpl(n.conf.Group),
@@ -333,6 +334,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	if n.apiV1 != "" {
 		return n.notifyV1(ctx, eventType, key, data, details, as...)
 	}
+
 	return n.notifyV2(ctx, eventType, key, data, details, as...)
 }
 
@@ -352,3 +354,17 @@ func errDetails(status int, body io.Reader) string {
 	}
 	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
 }
+
+func toCustomDetails(details map[string]string) map[string]interface{} {
+	customDetails := make(map[string]interface{}, len(details))
+	for k, v := range details {
+		var v2 interface{}
+		// Try to unmarshall any rendered templates as YAML.
+		if err := yaml.Unmarshal([]byte(v), &v2); err != nil {
+			customDetails[k] = v
+			continue
+		}
+		customDetails[k] = v2
+	}
+	return customDetails
+}

notify/pagerduty/pagerduty_test.go

@@ -23,6 +23,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -323,15 +324,18 @@ func TestErrDetails(t *testing.T) {
 }
 
 func TestEventSizeEnforcement(t *testing.T) {
-	bigDetails := map[string]string{
+	bigDetailsV1 := map[string]string{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]interface{}{
 		"firing": strings.Repeat("a", 513000),
 	}
 
 	// V1 Messages
 	msgV1 := &pagerDutyMessage{
 		ServiceKey: "01234567890123456789012345678901",
 		EventType:  "trigger",
-		Details:    bigDetails,
+		Details:    bigDetailsV1,
 	}
 
 	notifierV1, err := New(
@@ -353,7 +357,7 @@ func TestEventSizeEnforcement(t *testing.T) {
 		RoutingKey:  "01234567890123456789012345678901",
 		EventAction: "trigger",
 		Payload: &pagerDutyPayload{
-			CustomDetails: bigDetails,
+			CustomDetails: bigDetailsV2,
 		},
 	}
 
@@ -497,3 +501,28 @@ func TestPagerDutyEmptySrcHref(t *testing.T) {
 	}...)
 	require.NoError(t, err)
 }
+
+func Test_toCustomDetails(t *testing.T) {
+	type args struct {
+		details map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]interface{}
+	}{
+		{"number", args{map[string]string{"number": "123456789"}}, map[string]interface{}{"number": 123456789}},
+		{"string", args{map[string]string{"string": "test"}}, map[string]interface{}{"string": "test"}},
+		{"map", args{map[string]string{"map": "{key1: value1, key2: 2}"}}, map[string]interface{}{"map": map[string]interface{}{"key1": "value1", "key2": 2}}},
+		{"list_numbers", args{map[string]string{"list": "[1, 2, 3]"}}, map[string]interface{}{"list": []interface{}{1, 2, 3}}},
+		{"list_strings", args{map[string]string{"list": "[one, two, three]"}}, map[string]interface{}{"list": []interface{}{"one", "two", "three"}}},
+		{"nested", args{map[string]string{"parent": "child: {number: 123456789, string: test}"}}, map[string]interface{}{"parent": map[string]interface{}{"child": map[string]interface{}{"number": 123456789, "string": "test"}}}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := toCustomDetails(tt.args.details); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("toCustomDetails() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}