Skip to content

Commit 506264f

Browse files
jackgopack4jackgopack4
committed
add otel_collector payload transmit
1 parent c1201a0 commit 506264f

File tree

6 files changed

+164
-44
lines changed

6 files changed

+164
-44
lines changed

‎extension/datadogfleetautomationextension/fleetautomationextension.go‎

Lines changed: 66 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ type fleetAutomationExtension struct {
5858
mu sync.RWMutex
5959
uuid uuid.UUID
6060

61-
buildInfo component.BuildInfo
61+
buildInfo CustomBuildInfo
6262
moduleInfo service.ModuleInfos
6363
moduleInfoJSON *moduleInfoJSON
6464
activeComponentsJSON *activeComponentsJSON
@@ -70,6 +70,7 @@ type fleetAutomationExtension struct {
7070

7171
agentMetadataPayload AgentMetadata
7272
otelMetadataPayload OtelMetadata
73+
otelCollectorPayload OtelCollector
7374

7475
httpServer *http.Server
7576
healthCheckV2Enabled bool
@@ -123,6 +124,16 @@ func (e *fleetAutomationExtension) NotifyConfig(ctx context.Context, conf *confm
123124
fullConfig,
124125
)
125126

127+
e.otelCollectorPayload = prepareOtelCollectorPayload(
128+
e.hostname,
129+
e.hostnameSource,
130+
e.uuid.String(),
131+
e.version,
132+
e.extensionConfig.API.Site,
133+
fullConfig,
134+
e.buildInfo,
135+
)
136+
126137
// send payloads to Datadog backend
127138
_, err := e.prepareAndSendFleetAutomationPayloads()
128139
if err != nil {
@@ -197,31 +208,36 @@ func getHostname(ctx context.Context, providedHostname string, sp source.Provide
197208
func (e *fleetAutomationExtension) updateHostname(ctx context.Context) {
198209
// check for new hostname in extension config
199210
// TODO: switch to conf.Sub() method on refactor
200-
eCfg, err := e.collectorConfig.Sub(e.extensionID.String())
211+
extensionsConfig, err := e.collectorConfig.Sub(extensionsKind)
212+
if err != nil || len(extensionsConfig.AllKeys()) == 0 {
213+
e.telemetry.Logger.Error("Failed to get extensions config")
214+
return
215+
}
216+
eCfg, err := extensionsConfig.Sub(e.extensionID.String())
201217
if err != nil || len(eCfg.AllKeys()) == 0 {
202-
e.telemetry.Logger.Error("Failed to get extension config", zap.Error(err))
203-
} else {
204-
hostname := eCfg.Get("hostname")
205-
if hostname, ok := hostname.(string); ok {
206-
if hostname != "" {
207-
if hostname != e.hostname {
208-
e.hostname = hostname
209-
e.hostnameSource = "config"
210-
}
211-
} else {
212-
e.telemetry.Logger.Info("Hostname in config is empty, inferring hostname")
213-
hn, source, err := getHostname(ctx, e.hostname, e.hostnameProvider)
214-
if err != nil {
215-
e.telemetry.Logger.Error("Failed to infer hostname, collector will not show in Fleet Automation", zap.Error(err))
216-
e.hostname = ""
217-
e.hostnameSource = "unset"
218-
} else {
219-
e.hostname = hn
220-
e.telemetry.Logger.Info("Inferred hostname", zap.String("hostname", e.hostname))
221-
e.hostnameSource = source
222-
}
218+
e.telemetry.Logger.Error("Failed to get datadogfleetautomationextension config", zap.Error(err))
219+
return
220+
}
221+
hostname := eCfg.Get("hostname")
222+
if hostname, ok := hostname.(string); ok {
223+
if hostname != "" {
224+
if hostname != e.hostname {
225+
e.hostname = hostname
223226
}
227+
e.hostnameSource = "config"
228+
return
229+
}
230+
e.telemetry.Logger.Info("Hostname in config is empty, inferring hostname")
231+
hn, source, err := getHostname(ctx, e.hostname, e.hostnameProvider)
232+
if err != nil {
233+
e.telemetry.Logger.Error("Failed to infer hostname, collector will not show in Fleet Automation", zap.Error(err))
234+
e.hostname = ""
235+
e.hostnameSource = "unset"
236+
return
224237
}
238+
e.hostname = hn
239+
e.telemetry.Logger.Info("Inferred hostname", zap.String("hostname", e.hostname))
240+
e.hostnameSource = source
225241
}
226242
}
227243

@@ -307,6 +323,11 @@ func newExtension(
307323
compressor := newCompressor()
308324
serializer := newSerializer(forwarder, compressor, cfg, log, hostname)
309325
version := settings.BuildInfo.Version
326+
buildInfo := CustomBuildInfo{
327+
Command: settings.BuildInfo.Command,
328+
Description: settings.BuildInfo.Description,
329+
Version: settings.BuildInfo.Version,
330+
}
310331
return &fleetAutomationExtension{
311332
extensionID: settings.ID,
312333
extensionConfig: config,
@@ -315,7 +336,7 @@ func newExtension(
315336
forwarder: forwarder,
316337
compressor: &compressor,
317338
serializer: serializer,
318-
buildInfo: settings.BuildInfo,
339+
buildInfo: buildInfo,
319340
version: version,
320341
ticker: time.NewTicker(config.ReporterPeriod),
321342
done: make(chan bool),
@@ -354,3 +375,24 @@ func prepareOtelMetadataPayload(version, extensionVersion, command, fullConfig s
354375
FullConfiguration: fullConfig,
355376
}
356377
}
378+
379+
func prepareOtelCollectorPayload(
380+
hostname,
381+
hostnameSource,
382+
extensionUUID,
383+
version,
384+
site,
385+
fullConfig string,
386+
buildInfo CustomBuildInfo) OtelCollector {
387+
return OtelCollector{
388+
HostKey: "",
389+
Hostname: hostname,
390+
HostnameSource: hostnameSource,
391+
CollectorID: hostname + "-" + extensionUUID,
392+
CollectorVersion: version,
393+
ConfigSite: site,
394+
APIKeyUUID: "",
395+
BuildInfo: buildInfo,
396+
FullConfiguration: fullConfig,
397+
}
398+
}

‎extension/datadogfleetautomationextension/fleetautomationextension_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ func TestUpdateHostname(t *testing.T) {
498498
hostname: tt.initialHostname,
499499
hostnameProvider: mockProvider,
500500
extensionID: component.MustNewID(metadata.Type.String()),
501-
collectorConfig: confmap.NewFromStringMap(map[string]any{metadata.Type.String(): map[string]any{"hostname": tt.configHostname}}),
501+
collectorConfig: confmap.NewFromStringMap(map[string]any{extensionsKind: map[string]any{metadata.Type.String(): map[string]any{"hostname": tt.configHostname}}}),
502502
}
503503

504504
// Call updateHostname

‎extension/datadogfleetautomationextension/http.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,13 +151,16 @@ func makeGetRequest(uri string) (*http.Response, error) {
151151

152152
func (e *fleetAutomationExtension) prepareAndSendFleetAutomationPayloads() (*CombinedPayload, error) {
153153
// If health check v2 enabled, set Environment Variable Configuration to health check verbose query response
154+
var healthStatus string
154155
if e.healthCheckV2Enabled {
155156
componentStatus, err := e.getHealthCheckStatus()
156157
if err != nil {
157158
e.telemetry.Logger.Error("Failed to get health check status", zap.Error(err))
159+
healthStatus = ""
158160
} else {
159161
e.componentStatus = componentStatus
160-
e.otelMetadataPayload.EnvironmentVariableConfiguration = dataToFlattenedJSONString(e.componentStatus, false, false)
162+
healthStatus = dataToFlattenedJSONString(e.componentStatus, false, false)
163+
e.otelMetadataPayload.EnvironmentVariableConfiguration = healthStatus
161164
}
162165
}
163166

@@ -174,6 +177,13 @@ func (e *fleetAutomationExtension) prepareAndSendFleetAutomationPayloads() (*Com
174177
e.otelMetadataPayload.ProvidedConfiguration = dataToFlattenedJSONString(e.activeComponentsJSON, false, false) + "\n" + e.otelMetadataPayload.ProvidedConfiguration
175178
}
176179

180+
// add remaining data to otelCollectorPayload
181+
e.otelCollectorPayload.FullComponents = e.moduleInfoJSON.GetFullComponentsList()
182+
if e.activeComponentsJSON != nil {
183+
e.otelCollectorPayload.ActiveComponents = e.activeComponentsJSON.Components
184+
}
185+
e.otelCollectorPayload.HealthStatus = healthStatus
186+
177187
// Create the datadog_agent and the datadog_agent_otel payloads
178188
ap := agentPayload{
179189
Hostname: e.hostname,
@@ -188,6 +198,13 @@ func (e *fleetAutomationExtension) prepareAndSendFleetAutomationPayloads() (*Com
188198
UUID: e.uuid.String(),
189199
}
190200

201+
oc := otelCollectorPayload{
202+
Hostname: e.hostname,
203+
Timestamp: nowFunc().UnixNano(),
204+
Metadata: e.otelCollectorPayload,
205+
UUID: e.uuid.String(),
206+
}
207+
191208
// Use datadog-agent serializer to send these payloads
192209
if e.forwarder.State() == defaultforwarder.Started {
193210
err = e.serializer.SendMetadata(&ap)
@@ -198,13 +215,18 @@ func (e *fleetAutomationExtension) prepareAndSendFleetAutomationPayloads() (*Com
198215
if err != nil {
199216
return nil, fmt.Errorf("failed to send datadog_agent_otel payload: %w", err)
200217
}
218+
err = e.serializer.SendMetadata(&oc)
219+
if err != nil {
220+
return nil, fmt.Errorf("failed to send otel_collector payload: %w", err)
221+
}
201222
} else {
202223
e.telemetry.Logger.Warn("Forwarder is not started, skipping sending payloads")
203224
}
204225

205226
combinedPayload := &CombinedPayload{
206-
AgentPayload: ap,
207-
OtelPayload: p,
227+
CollectorPayload: oc,
228+
AgentPayload: ap,
229+
OtelPayload: p,
208230
}
209231
return combinedPayload, nil
210232
}

‎extension/datadogfleetautomationextension/http_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,29 @@ func TestPrepareAndSendFleetAutomationPayloads(t *testing.T) {
486486
}
487487

488488
const successfulInstanceResponse = `{
489+
"collector_payload": {
490+
"hostname": "test-hostname",
491+
"timestamp": 1741003200000000000,
492+
"otel_collector": {
493+
"host_key": "",
494+
"hostname": "",
495+
"hostname_source": "",
496+
"collector_id": "",
497+
"collector_version": "",
498+
"config_site": "",
499+
"api_key_uuid": "",
500+
"full_components": [],
501+
"active_components": null,
502+
"build_info": {
503+
"command": "",
504+
"description": "",
505+
"version": ""
506+
},
507+
"full_configuration": "",
508+
"health_status": ""
509+
},
510+
"uuid": "123e4567-e89b-12d3-a456-426614174000"
511+
},
489512
"otel_payload": {
490513
"hostname": "test-hostname",
491514
"timestamp": 1741003200000000000,

‎extension/datadogfleetautomationextension/payload.go‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,28 @@ import (
88
"fmt"
99

1010
"github.com/DataDog/datadog-agent/pkg/serializer/marshaler"
11-
"go.opentelemetry.io/collector/component"
1211
)
1312

13+
// CustomBuildInfo is a struct that duplicates the fields of component.BuildInfo with custom JSON tags
14+
type CustomBuildInfo struct {
15+
Command string `json:"command"`
16+
Description string `json:"description"`
17+
Version string `json:"version"`
18+
}
19+
1420
type OtelCollector struct {
15-
HostKey string `json:"host_key"`
16-
Hostname string `json:"hostname"`
17-
HostnameSource string `json:"hostname_source"`
18-
CollectorID string `json:"collector_id"`
19-
CollectorVersion string `json:"collector_version"`
20-
ConfigSite string `json:"config_site"`
21-
APIKeyUUID string `json:"api_key_uuid"`
22-
FullComponents []collectorModule `json:"full_components"`
23-
ActiveComponents []serviceComponent `json:"active_components"`
24-
BuildInfo component.BuildInfo `json:"build_info"`
25-
FullConfiguration string `json:"full_configuration"` // JSON passed as string
26-
HealthStatus string `json:"health_status"` // JSON passed as string
21+
HostKey string `json:"host_key"`
22+
Hostname string `json:"hostname"`
23+
HostnameSource string `json:"hostname_source"`
24+
CollectorID string `json:"collector_id"`
25+
CollectorVersion string `json:"collector_version"`
26+
ConfigSite string `json:"config_site"`
27+
APIKeyUUID string `json:"api_key_uuid"`
28+
FullComponents []collectorModule `json:"full_components"`
29+
ActiveComponents []serviceComponent `json:"active_components"`
30+
BuildInfo CustomBuildInfo `json:"build_info"`
31+
FullConfiguration string `json:"full_configuration"` // JSON passed as string
32+
HealthStatus string `json:"health_status"` // JSON passed as string
2733
}
2834

2935
type OtelMetadata struct {
@@ -39,8 +45,9 @@ type OtelMetadata struct {
3945
}
4046

4147
type CombinedPayload struct {
42-
OtelPayload otelAgentPayload `json:"otel_payload"`
43-
AgentPayload agentPayload `json:"agent_payload"`
48+
CollectorPayload otelCollectorPayload `json:"collector_payload"`
49+
OtelPayload otelAgentPayload `json:"otel_payload"`
50+
AgentPayload agentPayload `json:"agent_payload"`
4451
}
4552

4653
type AgentMetadata struct {
@@ -115,6 +122,13 @@ type agentPayload struct {
115122
UUID string `json:"uuid"`
116123
}
117124

125+
type otelCollectorPayload struct {
126+
Hostname string `json:"hostname"`
127+
Timestamp int64 `json:"timestamp"`
128+
Metadata OtelCollector `json:"otel_collector"`
129+
UUID string `json:"uuid"`
130+
}
131+
118132
type collectorModule struct {
119133
Type string `json:"type"`
120134
Kind string `json:"kind"`
@@ -177,6 +191,14 @@ func (m *moduleInfoJSON) MarshalJSON() ([]byte, error) {
177191
return json.Marshal(alias)
178192
}
179193

194+
func (m *moduleInfoJSON) GetFullComponentsList() []collectorModule {
195+
fullComponents := make([]collectorModule, 0, len(m.components))
196+
for _, comp := range m.components {
197+
fullComponents = append(fullComponents, comp)
198+
}
199+
return fullComponents
200+
}
201+
180202
type activeComponentsJSON struct {
181203
Components []serviceComponent `json:"active_components"`
182204
}
@@ -204,3 +226,14 @@ func (p *agentPayload) MarshalJSON() ([]byte, error) {
204226
func (p *agentPayload) SplitPayload(_ int) ([]marshaler.AbstractMarshaler, error) {
205227
return nil, fmt.Errorf("could not split inventories agent payload any more, payload is too big for intake")
206228
}
229+
230+
// MarshalJSON serializes a agentPayload to JSON
231+
func (p *otelCollectorPayload) MarshalJSON() ([]byte, error) {
232+
type collectorPayloadAlias otelCollectorPayload
233+
return json.Marshal((*collectorPayloadAlias)(p))
234+
}
235+
236+
// SplitPayload implements marshaler.AbstractMarshaler#SplitPayload.
237+
func (p *otelCollectorPayload) SplitPayload(_ int) ([]marshaler.AbstractMarshaler, error) {
238+
return nil, fmt.Errorf("could not split inventories agent payload any more, payload is too big for intake")
239+
}

‎extension/datadogfleetautomationextension/testdata/collector-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ extensions:
1717
datadogfleetautomation/test:
1818
api:
1919
key: ${env:DD_API_KEY}
20-
site: datadoghq.com
20+
site: datad0g.com
2121
hostname: "datadogfleetautomation-confighostname"
2222
reporter_period: 20m
2323
healthcheckv2:

0 commit comments

Comments
 (0)