Skip to content

Commit daf42da

Browse files
authored
macos app crash notable events (#54084)
<!--Please give us some feedback on your experience writing this PR ! https://app.datadoghq.com/forms/43db4c02-6837-400c-8083-692e141b1b88 !--> ### What does this PR do? This PR adds app crash collection on MacOS, reaching Windows parity. MacOS crash collection is split across two processes: - `system-probe` discovers system and per-user `Library/Logs/DiagnosticReports` directories, securely reads `.ips` files, sanitizes reports, and owns durable delivery state. - The core Agent, running as `_dd-agent`, polls system-probe over its filesystem-protected Unix socket, sends sanitized events to Event Management v2, and acknowledges events accepted by the Agent forwarding pipeline. ### Motivation [WINA-2932](https://datadoghq.atlassian.net/browse/WINA-2932) ### Describe how you validated your changes The validation is done by conducting local tests, unit tests, and end-to-end tests with backend, also ensuring all CI pipeline jobs pass. In particular, command `dda inv test` picks up `cmd/system-probe/modules/notable_events_darwin_test.go` in local manual test and the test is also wired to run in the merge queue (on main/release). Observing the notable events triggered by app crash was done with manually-triggered app crashes such as by running `python3 -c 'import ctypes; ctypes.string_at(0)'` and inspecting the Event Management dashboard. ### Additional Notes [WINA-2932]: https://datadoghq.atlassian.net/browse/WINA-2932?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ Co-authored-by: hongshi.guo <hongshi.guo@datadoghq.com>
1 parent 805bf9c commit daf42da

39 files changed

Lines changed: 8244 additions & 62 deletions

.github/CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,7 @@
522522
/pkg/jmxfetch/ @DataDog/agent-metric-pipelines
523523
/pkg/metrics/ @DataDog/agent-metric-pipelines
524524
/pkg/metrics/metricsource.go @DataDog/agent-metric-pipelines @DataDog/agent-integrations
525+
/pkg/notableevents/ @DataDog/windows-products
525526
/pkg/serializer/ @DataDog/agent-metric-pipelines
526527
/pkg/serializer/internal/metrics/origin_mapping.go @DataDog/agent-metric-pipelines @DataDog/agent-integrations
527528
/pkg/serverless/ @DataDog/serverless-azure-gcp

cmd/agent/subcommands/run/BUILD.bazel

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
name = "run",
88
srcs = [
99
"command.go",
10+
"command_darwin.go",
1011
"command_notwin.go",
1112
"command_snmptraps.go",
1213
"command_windows.go",
@@ -218,6 +219,7 @@ go_library(
218219
"//pkg/config/model",
219220
],
220221
"@rules_go//go/platform:darwin": [
222+
"//comp/notableevents/fx",
221223
"//comp/softwareinventory/fx",
222224
"//pkg/config/model",
223225
],
@@ -234,6 +236,7 @@ go_library(
234236
"//pkg/config/model",
235237
],
236238
"@rules_go//go/platform:ios": [
239+
"//comp/notableevents/fx",
237240
"//comp/softwareinventory/fx",
238241
"//pkg/config/model",
239242
],
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2026-present Datadog, Inc.
5+
6+
//go:build darwin
7+
8+
package run
9+
10+
import (
11+
"go.uber.org/fx"
12+
13+
notableeventsfx "github.com/DataDog/datadog-agent/comp/notableevents/fx"
14+
softwareinventoryfx "github.com/DataDog/datadog-agent/comp/softwareinventory/fx"
15+
)
16+
17+
// getPlatformModules returns the Darwin-specific fx modules for the Agent run command.
18+
func getPlatformModules() fx.Option {
19+
return fx.Options(
20+
softwareinventoryfx.Module(),
21+
notableeventsfx.Module(),
22+
)
23+
}

cmd/agent/subcommands/run/command_notwin.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// This product includes software developed at Datadog (https://www.datadoghq.com/).
44
// Copyright 2016-present Datadog, Inc.
55

6-
//go:build !windows
6+
//go:build !windows && !darwin
77

88
package run
99

cmd/system-probe/modules/BUILD.bazel

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ go_library(
2323
"network_tracer_usm.go",
2424
"network_tracer_windows.go",
2525
"noisy_neighbor.go",
26+
"notable_events_darwin.go",
2627
"oom_kill_probe.go",
2728
"ping.go",
2829
"privileged_logs_linux.go",
@@ -121,6 +122,7 @@ go_library(
121122
"//pkg/network/tracer",
122123
"//pkg/networkpath/payload",
123124
"//pkg/networkpath/traceroute/config",
125+
"//pkg/notableevents",
124126
"//pkg/system-probe/utils",
125127
"//pkg/util/log",
126128
"@com_github_datadog_datadog_traceroute//traceroute",
@@ -137,6 +139,7 @@ go_library(
137139
"//pkg/network/tracer",
138140
"//pkg/networkpath/payload",
139141
"//pkg/networkpath/traceroute/config",
142+
"//pkg/notableevents",
140143
"//pkg/system-probe/utils",
141144
"//pkg/util/log",
142145
"@com_github_datadog_datadog_traceroute//traceroute",
@@ -252,6 +255,7 @@ go_test(
252255
"gpu_test.go",
253256
"language_detection_test.go",
254257
"network_tracer_test.go",
258+
"notable_events_darwin_test.go",
255259
"process_test.go",
256260
"traceroute_test.go",
257261
],
@@ -292,6 +296,18 @@ go_test(
292296
"@com_github_stretchr_testify//require",
293297
"@org_golang_google_protobuf//proto",
294298
],
299+
"@rules_go//go/platform:darwin": [
300+
"//pkg/notableevents",
301+
"//pkg/system-probe/api/module",
302+
"@com_github_stretchr_testify//assert",
303+
"@com_github_stretchr_testify//require",
304+
],
305+
"@rules_go//go/platform:ios": [
306+
"//pkg/notableevents",
307+
"//pkg/system-probe/api/module",
308+
"@com_github_stretchr_testify//assert",
309+
"@com_github_stretchr_testify//require",
310+
],
295311
"@rules_go//go/platform:linux": [
296312
"//pkg/compliance/dbconfig",
297313
"//pkg/compliance/types",

cmd/system-probe/modules/modules.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var moduleOrder = []types.ModuleName{
3232
config.DiscoveryModule,
3333
config.GPUMonitoringModule, // GPU monitoring needs to be initialized after EventMonitor, so that we have the event consumer ready
3434
config.SoftwareInventoryModule,
35+
config.NotableEventsModule,
3536
config.PrivilegedLogsModule,
3637
config.InjectorModule,
3738
config.NoisyNeighborModule,
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2026-present Datadog, Inc.
5+
6+
//go:build darwin
7+
8+
package modules
9+
10+
import (
11+
"encoding/json"
12+
"errors"
13+
"fmt"
14+
"io"
15+
"net/http"
16+
"strings"
17+
18+
"github.com/DataDog/datadog-agent/pkg/notableevents"
19+
"github.com/DataDog/datadog-agent/pkg/system-probe/api/module"
20+
"github.com/DataDog/datadog-agent/pkg/system-probe/config"
21+
sysconfigtypes "github.com/DataDog/datadog-agent/pkg/system-probe/config/types"
22+
"github.com/DataDog/datadog-agent/pkg/system-probe/utils"
23+
"github.com/DataDog/datadog-agent/pkg/util/log"
24+
)
25+
26+
const (
27+
maxNotableEventsAckBodyBytes = 32 << 10
28+
maxNotableEventsAckIDs = 100
29+
maxNotableEventIDLength = 256
30+
)
31+
32+
type notableEventsCollector interface {
33+
Start() error
34+
Close() error
35+
Pending() []notableevents.Event
36+
Ack([]string) error
37+
}
38+
39+
// newNotableEventsCollector provides the platform collector constructor and a test seam.
40+
var newNotableEventsCollector = func() (notableEventsCollector, error) {
41+
return notableevents.NewCollector()
42+
}
43+
44+
// init registers the macOS notable-events module during system-probe startup.
45+
func init() { registerModule(NotableEvents) }
46+
47+
// NotableEvents is the Darwin notable-events module factory.
48+
var NotableEvents = &module.Factory{
49+
Name: config.NotableEventsModule,
50+
Fn: createNotableEventsModule,
51+
}
52+
53+
var _ module.Module = (*notableEventsModule)(nil)
54+
55+
type notableEventsModule struct {
56+
collector notableEventsCollector
57+
}
58+
59+
type notableEventsAckRequest struct {
60+
IDs []string `json:"ids"`
61+
}
62+
63+
type notableEventsErrorResponse struct {
64+
Error string `json:"error"`
65+
}
66+
67+
// createNotableEventsModule constructs and starts the collector backing the module.
68+
func createNotableEventsModule(_ *sysconfigtypes.Config, _ module.FactoryDependencies) (module.Module, error) {
69+
collector, err := newNotableEventsCollector()
70+
if err != nil {
71+
return nil, fmt.Errorf("create notable events collector: %w", err)
72+
}
73+
if err := collector.Start(); err != nil {
74+
if closeErr := collector.Close(); closeErr != nil {
75+
log.Warnf("Failed to close notable events collector after start failure: %v", closeErr)
76+
}
77+
return nil, fmt.Errorf("start notable events collector: %w", err)
78+
}
79+
return &notableEventsModule{collector: collector}, nil
80+
}
81+
82+
// Register exposes the pending-event and acknowledgement endpoints.
83+
func (m *notableEventsModule) Register(router *module.Router) error {
84+
router.HandleFunc("GET /check", m.handleCheck)
85+
router.HandleFunc("POST /ack", m.handleAck)
86+
return nil
87+
}
88+
89+
// handleCheck returns a snapshot of events awaiting forwarding-pipeline acceptance.
90+
func (m *notableEventsModule) handleCheck(w http.ResponseWriter, req *http.Request) {
91+
writeNotableEventsJSON(req, w, http.StatusOK, m.collector.Pending())
92+
}
93+
94+
// handleAck validates and persists acknowledgement of accepted events.
95+
func (m *notableEventsModule) handleAck(w http.ResponseWriter, req *http.Request) {
96+
body, status, err := decodeNotableEventsAckRequest(w, req)
97+
if err != nil {
98+
writeNotableEventsJSON(req, w, status, notableEventsErrorResponse{Error: err.Error()})
99+
return
100+
}
101+
if err := m.collector.Ack(body.IDs); err != nil {
102+
log.Errorf("Failed to acknowledge notable events: %v", err)
103+
writeNotableEventsJSON(req, w, http.StatusInternalServerError, notableEventsErrorResponse{Error: "failed to acknowledge notable events"})
104+
return
105+
}
106+
writeNotableEventsJSON(req, w, http.StatusOK, struct{}{})
107+
}
108+
109+
// decodeNotableEventsAckRequest decodes a bounded acknowledgement request and validates its IDs.
110+
func decodeNotableEventsAckRequest(w http.ResponseWriter, req *http.Request) (notableEventsAckRequest, int, error) {
111+
var body notableEventsAckRequest
112+
req.Body = http.MaxBytesReader(w, req.Body, maxNotableEventsAckBodyBytes)
113+
decoder := json.NewDecoder(req.Body)
114+
decoder.DisallowUnknownFields()
115+
116+
if err := decoder.Decode(&body); err != nil {
117+
status, decodeErr := notableEventsDecodeError(err)
118+
return body, status, decodeErr
119+
}
120+
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
121+
if err == nil {
122+
err = errors.New("multiple JSON values")
123+
}
124+
status, decodeErr := notableEventsDecodeError(err)
125+
return body, status, decodeErr
126+
}
127+
if body.IDs == nil {
128+
return body, http.StatusBadRequest, errors.New("ids is required")
129+
}
130+
if len(body.IDs) > maxNotableEventsAckIDs {
131+
return body, http.StatusBadRequest, fmt.Errorf("ids must contain at most %d entries", maxNotableEventsAckIDs)
132+
}
133+
for _, id := range body.IDs {
134+
if strings.TrimSpace(id) == "" {
135+
return body, http.StatusBadRequest, errors.New("ids must not contain empty values")
136+
}
137+
if len(id) > maxNotableEventIDLength {
138+
return body, http.StatusBadRequest, fmt.Errorf("ids must not exceed %d bytes", maxNotableEventIDLength)
139+
}
140+
}
141+
return body, http.StatusOK, nil
142+
}
143+
144+
// notableEventsDecodeError maps JSON decoding failures to safe HTTP responses.
145+
func notableEventsDecodeError(err error) (int, error) {
146+
var maxBytesErr *http.MaxBytesError
147+
if errors.As(err, &maxBytesErr) {
148+
return http.StatusRequestEntityTooLarge, fmt.Errorf("request body exceeds %d bytes", maxNotableEventsAckBodyBytes)
149+
}
150+
return http.StatusBadRequest, fmt.Errorf("invalid JSON request: %w", err)
151+
}
152+
153+
// writeNotableEventsJSON writes a compact JSON response with the requested status.
154+
func writeNotableEventsJSON(req *http.Request, w http.ResponseWriter, status int, body interface{}) {
155+
w.Header().Set("Content-Type", "application/json")
156+
w.WriteHeader(status)
157+
utils.WriteAsJSON(req, w, body, utils.CompactOutput)
158+
}
159+
160+
// GetStats returns the module's currently empty runtime statistics.
161+
func (m *notableEventsModule) GetStats() map[string]interface{} {
162+
return map[string]interface{}{}
163+
}
164+
165+
// Close releases the collector and its filesystem monitoring resources.
166+
func (m *notableEventsModule) Close() {
167+
if err := m.collector.Close(); err != nil {
168+
log.Warnf("Failed to close notable events collector: %v", err)
169+
}
170+
}

0 commit comments

Comments
 (0)