fix: pageduty custom_details in payload

This change fixes the `custom_details` value in PagerDuty V2 API payloads.

Fixes #2477
Fixes #3218

Alertmanager PagerDuty configuration allows users to define `details` under `pagerduty_configs`:
```yaml
[ details: { <string>: <tmpl_string>, ... } | default = {
  firing:       '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
  resolved:     '{{ template "pagerduty.default.instances" .Alerts.Resolved }}'
  num_firing:   '{{ .Alerts.Firing | len }}'
  num_resolved: '{{ .Alerts.Resolved | len }}'
} ]
```

The internal Alertmanager configuration structure is defined as:
```go
// PagerdutyConfig configures notifications via PagerDuty.
type PagerdutyConfig struct {
...
	Details        map[string]string `yaml:"details,omitempty" json:"details,omitempty"`
}
```
And the PagerDuty payload is defined as:
```go
type pagerDutyPayload struct {
...
	CustomDetails map[string]string `json:"custom_details,omitempty"`
```

The current flow of configuration parsing and encoding is:
1. `pagerduty_configs[].details` are read from configuration as `map[string]string`
2. The `details` map values are parsed as text templates
3. The result is passed as `custom_details`, part of the JSON payload sent to PagerDuty API V2

Here is an example payload using `.Alerts.Firing`, which returns a list of currently firing alert objects in this group.
Documented here: https://prometheus.io/docs/alerting/latest/notifications/#data
This results in a payload like below:
```json
{
  "client": "Alertmanager",
...
  "custom_details": {
    "firing": "Labels:
 - alertname = Server_Down
...
Annotations:
 - summary = Server is down
...
"
    }
}
```
Using `map[string]string` in payloads cause the rendered YAML templates to be encoded as JSON strings.
This is undesirable in most cases as the end-user might expect a nested JSON object which is machine readable/parsable by it's fields.

This change:
- adds logic to unmarshall `details` values as YAML with fallback to string
- uses `map[string]interface{}` to encode `custom_details` to JSON
- requires gopkg.in/yaml.v3, see go-yaml/yaml#591
- does not change existing yaml.v2 dependency, see #3322

Signed-off-by: Siavash Safi <[email protected]>

Author: siavashs

go.mod

@@ -48,6 +48,7 @@ require (
 	golang.org/x/tools v0.22.0
 	gopkg.in/telebot.v3 v3.3.6
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -100,5 +101,4 @@ require (
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

notify/pagerduty/pagerduty.go

@@ -29,6 +29,7 @@ import (
 	"github.com/go-kit/log/level"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"gopkg.in/yaml.v3"
 
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
@@ -107,14 +108,14 @@ type pagerDutyImage struct {
 }
 
 type pagerDutyPayload struct {
-	Summary       string            `json:"summary"`
-	Source        string            `json:"source"`
-	Severity      string            `json:"severity"`
-	Timestamp     string            `json:"timestamp,omitempty"`
-	Class         string            `json:"class,omitempty"`
-	Component     string            `json:"component,omitempty"`
-	Group         string            `json:"group,omitempty"`
-	CustomDetails map[string]string `json:"custom_details,omitempty"`
+	Summary       string                 `json:"summary"`
+	Source        string                 `json:"source"`
+	Severity      string                 `json:"severity"`
+	Timestamp     string                 `json:"timestamp,omitempty"`
+	Class         string                 `json:"class,omitempty"`
+	Component     string                 `json:"component,omitempty"`
+	Group         string                 `json:"group,omitempty"`
+	CustomDetails map[string]interface{} `json:"custom_details,omitempty"`
 }
 
 func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
@@ -129,7 +130,7 @@ func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
 		if n.apiV1 != "" {
 			msg.Details = map[string]string{"error": truncatedMsg}
 		} else {
-			msg.Payload.CustomDetails = map[string]string{"error": truncatedMsg}
+			msg.Payload.CustomDetails = map[string]interface{}{"error": truncatedMsg}
 		}
 
 		warningMsg := fmt.Sprintf("Truncated Details because message of size %s exceeds limit %s", units.MetricBytes(buf.Len()).String(), units.MetricBytes(maxEventSize).String())
@@ -210,7 +211,7 @@ func (n *Notifier) notifyV2(
 	eventType string,
 	key notify.Key,
 	data *template.Data,
-	details map[string]string,
+	details map[string]interface{},
 	as ...*types.Alert,
 ) (bool, error) {
 	var tmplErr error
@@ -333,7 +334,19 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	if n.apiV1 != "" {
 		return n.notifyV1(ctx, eventType, key, data, details, as...)
 	}
-	return n.notifyV2(ctx, eventType, key, data, details, as...)
+
+	customDetails := make(map[string]interface{}, len(n.conf.Details))
+	for k, v := range details {
+		var v2 interface{}
+		// Try to unmarshall any rendered templates as YAML.
+		if err := yaml.Unmarshal([]byte(v), &v2); err != nil {
+			customDetails[k] = v
+			continue
+		}
+		customDetails[k] = v2
+	}
+
+	return n.notifyV2(ctx, eventType, key, data, customDetails, as...)
 }
 
 func errDetails(status int, body io.Reader) string {

notify/pagerduty/pagerduty_test.go

@@ -323,15 +323,18 @@ func TestErrDetails(t *testing.T) {
 }
 
 func TestEventSizeEnforcement(t *testing.T) {
-	bigDetails := map[string]string{
+	bigDetailsV1 := map[string]string{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]interface{}{
 		"firing": strings.Repeat("a", 513000),
 	}
 
 	// V1 Messages
 	msgV1 := &pagerDutyMessage{
 		ServiceKey: "01234567890123456789012345678901",
 		EventType:  "trigger",
-		Details:    bigDetails,
+		Details:    bigDetailsV1,
 	}
 
 	notifierV1, err := New(
@@ -353,7 +356,7 @@ func TestEventSizeEnforcement(t *testing.T) {
 		RoutingKey:  "01234567890123456789012345678901",
 		EventAction: "trigger",
 		Payload: &pagerDutyPayload{
-			CustomDetails: bigDetails,
+			CustomDetails: bigDetailsV2,
 		},
 	}