|
| 1 | +// Unless explicitly stated otherwise all files in this repository are licensed |
| 2 | +// under the Apache License Version 2.0. |
| 3 | +// This product includes software developed at Datadog (https://www.datadoghq.com/). |
| 4 | +// Copyright 2016-present Datadog, Inc. |
| 5 | + |
| 6 | +package config |
| 7 | + |
| 8 | +import ( |
| 9 | + "context" |
| 10 | + "testing" |
| 11 | + |
| 12 | + "github.com/stretchr/testify/assert" |
| 13 | + "github.com/stretchr/testify/require" |
| 14 | + |
| 15 | + configmock "github.com/DataDog/datadog-agent/pkg/config/mock" |
| 16 | + "github.com/DataDog/datadog-agent/pkg/util/compression" |
| 17 | +) |
| 18 | + |
| 19 | +// TestDDOTCompressionConsistency guards the invariant that the DDOT (otel-agent) |
| 20 | +// exporter compresses every signal with a single algorithm — zstd — so the |
| 21 | +// per-signal compressors cannot silently diverge. |
| 22 | +// |
| 23 | +// Metrics and logs are config-driven: their algorithm and level come from the |
| 24 | +// agent-config keys set in NewConfigComponent, asserted here directly. |
| 25 | +// |
| 26 | +// Traces are NOT config-driven: cmd/otel-agent/subcommands/run/command.go wires |
| 27 | +// comp/trace/compression/fx-zstd (zstd at BestSpeed; the level is intentionally |
| 28 | +// fixed for traces). "zstd" below is therefore the shared invariant all three |
| 29 | +// signals must satisfy. The on-the-wire guard that traces actually ship zstd |
| 30 | +// lives in the OTLP integration test (Content-Encoding assertion). |
| 31 | +func TestDDOTCompressionConsistency(t *testing.T) { |
| 32 | + configmock.New(t) |
| 33 | + c, err := NewConfigComponent(context.Background(), "", []string{"testdata/config_default.yaml"}) |
| 34 | + require.NoError(t, err) |
| 35 | + |
| 36 | + const wantAlgo = "zstd" |
| 37 | + const wantLevel = ddotZstdCompressionLevel |
| 38 | + |
| 39 | + metricsKind := c.GetString("serializer_compressor_kind") |
| 40 | + logsKind := c.GetString("logs_config.compression_kind") |
| 41 | + |
| 42 | + // All signals share one algorithm by default. |
| 43 | + assert.Equal(t, wantAlgo, metricsKind, "metrics must default to zstd") |
| 44 | + assert.Equal(t, wantAlgo, logsKind, "logs must default to zstd") |
| 45 | + assert.Equal(t, metricsKind, logsKind, "metrics and logs compression algorithms must not diverge") |
| 46 | + |
| 47 | + // logs_config.compression_kind must be IsConfigured() (set at a non-default source) |
| 48 | + // so the logs pipeline does NOT fall back to gzip when logs_config.additional_endpoints |
| 49 | + // is set. additional_endpoints are other Datadog endpoints (multi-region / dual-ship / |
| 50 | + // MRF) that accept zstd, so every log endpoint should ship zstd — matching the metrics |
| 51 | + // forwarder, which already fans one zstd payload to all endpoints. |
| 52 | + assert.True(t, c.IsConfigured("logs_config.compression_kind"), |
| 53 | + "logs_config.compression_kind must be configured so the additional_endpoints gzip fallback is bypassed") |
| 54 | + |
| 55 | + // Levels default to 3 for the signals that support a configurable level. |
| 56 | + assert.Equal(t, wantLevel, c.GetInt("serializer_zstd_compressor_level"), "metrics zstd level should default to 3") |
| 57 | + assert.Equal(t, wantLevel, c.GetInt("logs_config.zstd_compression_level"), "logs zstd level should default to 3") |
| 58 | + |
| 59 | + // DDOT deliberately stays on the v2 metrics intake (v3 is a separate effort). |
| 60 | + assert.Equal(t, "false", c.GetString("use_v3_api.series.enabled"), "DDOT must stay on the v2 metrics intake") |
| 61 | +} |
| 62 | + |
| 63 | +// TestDDOTCompressionLevelOverridable verifies the per-signal compression level |
| 64 | +// stays overridable (e.g. via DD_* env vars) rather than being forced, so the |
| 65 | +// SourceDefault precedence chosen in NewConfigComponent does not lock operators out. |
| 66 | +func TestDDOTCompressionLevelOverridable(t *testing.T) { |
| 67 | + configmock.New(t) |
| 68 | + t.Setenv("DD_SERIALIZER_ZSTD_COMPRESSOR_LEVEL", "6") |
| 69 | + t.Setenv("DD_LOGS_CONFIG_ZSTD_COMPRESSION_LEVEL", "9") |
| 70 | + |
| 71 | + c, err := NewConfigComponent(context.Background(), "", []string{"testdata/config_default.yaml"}) |
| 72 | + require.NoError(t, err) |
| 73 | + |
| 74 | + assert.Equal(t, 6, c.GetInt("serializer_zstd_compressor_level"), "metrics zstd level should be overridable via env") |
| 75 | + assert.Equal(t, 9, c.GetInt("logs_config.zstd_compression_level"), "logs zstd level should be overridable via env") |
| 76 | + |
| 77 | + // Overriding only the level must not change the algorithm. |
| 78 | + assert.Equal(t, "zstd", c.GetString("serializer_compressor_kind")) |
| 79 | + assert.Equal(t, "zstd", c.GetString("logs_config.compression_kind")) |
| 80 | +} |
| 81 | + |
| 82 | +// TestDDOTHostMetadataCompression explicitly guards that host metadata — a |
| 83 | +// separate submission path from metrics — is compressed with zstd. |
| 84 | +// |
| 85 | +// The DD exporter pushes host metadata through the agent serializer: |
| 86 | +// serializer.SendHostMetadata -> sendMetadata -> split.CheckSizeAndSerialize(m, |
| 87 | +// true /*compress*/, s.Strategy). s.Strategy is the serializer's compressor, built |
| 88 | +// from the SAME serializer_compressor_kind / serializer_zstd_compressor_level keys |
| 89 | +// as the metrics series/sketches path — there is no dedicated host-metadata |
| 90 | +// compression knob. So host metadata is zstd (level 3) whenever metrics are; this |
| 91 | +// test pins that so the two cannot silently diverge. |
| 92 | +func TestDDOTHostMetadataCompression(t *testing.T) { |
| 93 | + configmock.New(t) |
| 94 | + c, err := NewConfigComponent(context.Background(), "", []string{"testdata/config_default.yaml"}) |
| 95 | + require.NoError(t, err) |
| 96 | + |
| 97 | + assert.Equal(t, "zstd", c.GetString("serializer_compressor_kind"), |
| 98 | + "host metadata shares the metrics compressor (serializer_compressor_kind); both must be zstd") |
| 99 | + assert.Equal(t, ddotZstdCompressionLevel, c.GetInt("serializer_zstd_compressor_level"), |
| 100 | + "host metadata uses the metrics zstd level") |
| 101 | +} |
| 102 | + |
| 103 | +// TestDDOTSupportedCompressors covers every compression algorithm DDOT supports |
| 104 | +// per signal — not just the zstd default — on two axes: |
| 105 | +// |
| 106 | +// 1. Selectability: each supported algorithm can be chosen via the DDOT config |
| 107 | +// surface (DD_* env override beats the SourceDefault/SourceFile the |
| 108 | +// otel-agent sets), so operators are not locked into zstd. |
| 109 | +// 2. Wire encoding: the Content-Encoding each algorithm emits — the value |
| 110 | +// fakeintake/the intake observe and the e2e utils.TestCompression asserts — |
| 111 | +// is pinned here. Note zlib ships as "deflate" (not "zlib") and none ships |
| 112 | +// as "identity"; those non-obvious mappings are the main regression risk. |
| 113 | +// |
| 114 | +// Metrics (serializer_compressor_kind, shared by series/sketches/host-metadata) |
| 115 | +// support zstd/zlib/gzip/none. Logs (logs_config.compression_kind) support |
| 116 | +// zstd/gzip, used verbatim as the Content-Encoding. Traces are compile-time |
| 117 | +// (the fx-zstd/fx-gzip module in command.go), not a runtime config key, so they |
| 118 | +// are out of scope for this config-level test; the OTLP integration test proves |
| 119 | +// traces actually ship valid zstd on the wire. |
| 120 | +func TestDDOTSupportedCompressors(t *testing.T) { |
| 121 | + // Pin the on-the-wire Content-Encoding for each algorithm. These are the |
| 122 | + // exact strings e2e/intake see; changing one would silently break the e2e |
| 123 | + // compression assertions, so guard them here. |
| 124 | + assert.Equal(t, "zstd", compression.ZstdEncoding, "zstd must ship Content-Encoding zstd") |
| 125 | + assert.Equal(t, "deflate", compression.ZlibEncoding, "zlib must ship Content-Encoding deflate (not \"zlib\")") |
| 126 | + assert.Equal(t, "gzip", compression.GzipEncoding, "gzip must ship Content-Encoding gzip") |
| 127 | + |
| 128 | + // metrics: every supported serializer_compressor_kind must be selectable, |
| 129 | + // and each maps to the wire encoding recorded above (none -> "identity"). |
| 130 | + metrics := []struct{ kind, wantEncoding string }{ |
| 131 | + {compression.ZstdKind, compression.ZstdEncoding}, |
| 132 | + {compression.ZlibKind, compression.ZlibEncoding}, |
| 133 | + {compression.GzipKind, compression.GzipEncoding}, |
| 134 | + {compression.NoneKind, "identity"}, |
| 135 | + } |
| 136 | + for _, tc := range metrics { |
| 137 | + t.Run("metrics/"+tc.kind, func(t *testing.T) { |
| 138 | + configmock.New(t) |
| 139 | + t.Setenv("DD_SERIALIZER_COMPRESSOR_KIND", tc.kind) |
| 140 | + c, err := NewConfigComponent(context.Background(), "", []string{"testdata/config_default.yaml"}) |
| 141 | + require.NoError(t, err) |
| 142 | + assert.Equalf(t, tc.kind, c.GetString("serializer_compressor_kind"), |
| 143 | + "operators must be able to select %q for metrics via DD_SERIALIZER_COMPRESSOR_KIND (wire encoding %q)", tc.kind, tc.wantEncoding) |
| 144 | + }) |
| 145 | + } |
| 146 | + |
| 147 | + // logs: only zstd and gzip are supported; the pipeline uses the kind |
| 148 | + // verbatim as the Content-Encoding, so the kind is the wire value. |
| 149 | + logs := []string{compression.ZstdKind, compression.GzipKind} |
| 150 | + for _, kind := range logs { |
| 151 | + t.Run("logs/"+kind, func(t *testing.T) { |
| 152 | + configmock.New(t) |
| 153 | + t.Setenv("DD_LOGS_CONFIG_COMPRESSION_KIND", kind) |
| 154 | + c, err := NewConfigComponent(context.Background(), "", []string{"testdata/config_default.yaml"}) |
| 155 | + require.NoError(t, err) |
| 156 | + assert.Equalf(t, kind, c.GetString("logs_config.compression_kind"), |
| 157 | + "operators must be able to select %q for logs via DD_LOGS_CONFIG_COMPRESSION_KIND", kind) |
| 158 | + assert.Truef(t, c.IsConfigured("logs_config.compression_kind"), |
| 159 | + "overriding logs_config.compression_kind to %q must keep it IsConfigured() so the additional_endpoints gzip fallback stays bypassed", kind) |
| 160 | + }) |
| 161 | + } |
| 162 | +} |
0 commit comments