Skip to content

Commit e5e8e68

Browse files
committed
Introduce a CI job that test the config generated code
1 parent 5d839f7 commit e5e8e68

9 files changed

Lines changed: 410 additions & 1 deletion

File tree

.gitlab/JOBOWNERS

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ build_windows_container_entrypoint @DataDog/windows-products
5757
generate_config_schema-linux @DataDog/fleet-automation
5858
generate_config_schema-macos @DataDog/fleet-automation
5959
generate_config_schema-windows @DataDog/fleet-automation
60-
# Cross-compiled binary builds for validation purpose
60+
check_config_codegen_drift-linux @DataDog/fleet-automation
61+
check_config_codegen_drift-macos @DataDog/fleet-automation
62+
check_config_codegen_drift-windows @DataDog/fleet-automation
6163
cross_build_agent-binary_aix_ppc64 @DataDog/agent-build
6264
cross_build_trace_agent-binary_aix_ppc64 @DataDog/agent-build
6365

.gitlab/build/binary_build/schema_generation.yml

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,90 @@ generate_config_schema-windows:
100100
expire_in: 2 weeks
101101
paths:
102102
- $CI_PROJECT_DIR/pkg/config/schema/yaml/*.yaml
103+
104+
# Verify that regenerating the config settings code (`dda inv schema.codegen`)
105+
# does not change the Agent's runtime configuration defaults. See the
106+
# schema.check-codegen-drift invoke task for the full flow.
107+
.check_config_codegen_drift:
108+
stage: binary_build
109+
rules:
110+
- !reference [.except_mergequeue]
111+
- !reference [.on_main_or_release_branch]
112+
- !reference [.on_schema_paths_changed]
113+
114+
check_config_codegen_drift-linux:
115+
extends:
116+
- .check_config_codegen_drift
117+
- .bazel:defs:cache:progressive
118+
variables:
119+
KUBERNETES_CPU_REQUEST: 16
120+
KUBERNETES_MEMORY_REQUEST: 16Gi
121+
KUBERNETES_MEMORY_LIMIT: 16Gi
122+
TEST_OUTPUT_FILE: test_output
123+
image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
124+
tags: ["arch:amd64", "specific:true"]
125+
needs: ["go_deps"]
126+
before_script:
127+
- !reference [.retrieve_linux_go_deps]
128+
script:
129+
- dda inv -- schema.check-codegen-drift --output-dir $CI_PROJECT_DIR/config_codegen_drift
130+
artifacts:
131+
when: always
132+
expire_in: 2 weeks
133+
paths:
134+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
135+
136+
check_config_codegen_drift-macos:
137+
extends:
138+
- .check_config_codegen_drift
139+
- .bazel:defs:cache:macos
140+
- .macos_gitlab
141+
variables:
142+
KUBERNETES_CPU_REQUEST: 16
143+
KUBERNETES_MEMORY_REQUEST: 16Gi
144+
KUBERNETES_MEMORY_LIMIT: 16Gi
145+
TEST_OUTPUT_FILE: test_output
146+
tags: ["macos:sonoma-amd64", "specific:true"]
147+
needs: ["go_deps", "go_tools_deps"]
148+
script:
149+
- dda inv -- schema.check-codegen-drift --output-dir $CI_PROJECT_DIR/config_codegen_drift
150+
artifacts:
151+
when: always
152+
expire_in: 2 weeks
153+
paths:
154+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
155+
156+
check_config_codegen_drift-windows:
157+
extends:
158+
- .check_config_codegen_drift
159+
- .bazel:defs:cache:windows
160+
- .windows_docker_default
161+
needs: ["go_deps", "go_tools_deps"]
162+
variables:
163+
ARCH: "x64"
164+
script:
165+
- $ErrorActionPreference = "Stop"
166+
- !reference [.sanitize_goproxy_windows]
167+
- >
168+
.\tools\ci\docker-run-with-bazel-cache.ps1
169+
-m 24576M
170+
-v "$(Get-Location):c:\mnt"
171+
-e GITLAB_CI
172+
-e CI_JOB_ID
173+
-e CI_PIPELINE_ID
174+
-e CI_PROJECT_NAME
175+
-e AWS_NETWORKING=true
176+
-e GOMODCACHE="c:\modcache"
177+
-e GOPROXY
178+
-e GONOSUMDB
179+
-e PIP_INDEX_URL
180+
-e DDA_FEATURE_FLAGS_CI_SSM_KEY_WINDOWS
181+
-e CI_IDENTITIES_GITLAB_ID_TOKEN
182+
${WINBUILDIMAGE}
183+
powershell.exe -c "c:\mnt\tasks\winbuildscripts\Check-ConfigCodegenDrift.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1"
184+
- If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
185+
artifacts:
186+
when: always
187+
expire_in: 2 weeks
188+
paths:
189+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
load("@rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "dumpconfig",
5+
srcs = ["command.go"],
6+
importpath = "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dumpconfig",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//cmd/agent/command",
10+
"//pkg/config/model",
11+
"//pkg/config/setup",
12+
"@com_github_spf13_cobra//:cobra",
13+
],
14+
)
15+
16+
go_test(
17+
name = "dumpconfig_test",
18+
srcs = ["command_test.go"],
19+
embed = [":dumpconfig"],
20+
gotags = ["test"],
21+
deps = [
22+
"//cmd/agent/command",
23+
"@com_github_stretchr_testify//assert",
24+
"@com_github_stretchr_testify//require",
25+
],
26+
)
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
// Package dumpconfig implements 'agent dumpconfig'.
7+
//
8+
// It initializes the configuration defaults (without reading any datadog.yaml
9+
// or system-probe.yaml file) and prints the resulting runtime configuration as
10+
// JSON. It is used by CI to verify that regenerating the config settings code
11+
// (`dda inv schema.codegen`) does not change the Agent's runtime defaults.
12+
package dumpconfig
13+
14+
import (
15+
"encoding/json"
16+
"fmt"
17+
"time"
18+
19+
"github.com/spf13/cobra"
20+
21+
"github.com/DataDog/datadog-agent/cmd/agent/command"
22+
"github.com/DataDog/datadog-agent/pkg/config/model"
23+
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
24+
)
25+
26+
// cliParams are the command-line arguments for this subcommand
27+
type cliParams struct {
28+
*command.GlobalParams
29+
Target string
30+
}
31+
32+
// Commands returns a slice of subcommands for the 'agent' command.
33+
func Commands(globalParams *command.GlobalParams) []*cobra.Command {
34+
cliParams := &cliParams{
35+
GlobalParams: globalParams,
36+
}
37+
38+
dumpConfigCommand := &cobra.Command{
39+
Use: "dumpconfig",
40+
Short: "Dump the runtime configuration defaults as JSON",
41+
Long: ``,
42+
Hidden: true,
43+
RunE: func(_ *cobra.Command, _ []string) error {
44+
return run(cliParams)
45+
},
46+
}
47+
dumpConfigCommand.Flags().StringVar(&cliParams.Target, "target", "", "config to dump: core or system-probe")
48+
49+
return []*cobra.Command{dumpConfigCommand}
50+
}
51+
52+
func run(cliParams *cliParams) error {
53+
// Initialize the global config objects with their defaults only. No
54+
// datadog.yaml or system-probe.yaml file is loaded, so what we dump is the
55+
// pure set of runtime defaults.
56+
pkgconfigsetup.InitConfigObjects()
57+
58+
var cfg model.Config
59+
switch cliParams.Target {
60+
case "core":
61+
cfg = pkgconfigsetup.Datadog()
62+
case "system-probe":
63+
cfg = pkgconfigsetup.SystemProbe()
64+
default:
65+
return fmt.Errorf("unknown target '%s', valid ones are 'core' or 'system-probe'", cliParams.Target)
66+
}
67+
68+
// json.Marshal sorts map keys, so the output is deterministic across runs.
69+
data, err := json.MarshalIndent(durationsToString(cfg.AllSettings()), "", " ")
70+
if err != nil {
71+
return err
72+
}
73+
fmt.Println(string(data))
74+
return nil
75+
}
76+
77+
// durationsToString recursively walks a value decoded from the config and
78+
// replaces every time.Duration with its string representation (e.g. "10s"),
79+
// so durations are dumped as human-readable strings instead of raw
80+
// nanosecond integers.
81+
func durationsToString(v interface{}) interface{} {
82+
switch val := v.(type) {
83+
case time.Duration:
84+
return val.String()
85+
case map[string]interface{}:
86+
for k, elem := range val {
87+
val[k] = durationsToString(elem)
88+
}
89+
return val
90+
case []interface{}:
91+
for i, elem := range val {
92+
val[i] = durationsToString(elem)
93+
}
94+
return val
95+
default:
96+
return v
97+
}
98+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
package dumpconfig
7+
8+
import (
9+
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
14+
"github.com/DataDog/datadog-agent/cmd/agent/command"
15+
)
16+
17+
func TestRun(t *testing.T) {
18+
for _, target := range []string{"core", "system-probe"} {
19+
t.Run(target, func(t *testing.T) {
20+
require.NoError(t, run(&cliParams{GlobalParams: &command.GlobalParams{}, Target: target}))
21+
})
22+
}
23+
}
24+
25+
func TestRunUnknownTarget(t *testing.T) {
26+
err := run(&cliParams{GlobalParams: &command.GlobalParams{}, Target: "unknown"})
27+
assert.ErrorContains(t, err, "unknown target")
28+
}

cmd/agent/subcommands/subcommands.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
cmddogstatsdcapture "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdcapture"
2121
cmddogstatsdreplay "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdreplay"
2222
cmddogstatsdstats "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdstats"
23+
cmddumpconfig "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dumpconfig"
2324
cmdexperimental "github.com/DataDog/datadog-agent/cmd/agent/subcommands/experimental"
2425
cmdflare "github.com/DataDog/datadog-agent/cmd/agent/subcommands/flare"
2526
cmdhealth "github.com/DataDog/datadog-agent/cmd/agent/subcommands/health"
@@ -60,6 +61,7 @@ func AgentSubcommands() []command.SubcommandFactory {
6061
cmddogstatsdcapture.Commands,
6162
cmddogstatsdreplay.Commands,
6263
cmddogstatsdstats.Commands,
64+
cmddumpconfig.Commands,
6365
cmdflare.Commands,
6466
cmdhealth.Commands,
6567
cmdhostname.Commands,

tasks/schema/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
from invoke.collection import Collection
22

33
from tasks.schema.add_setting import add_setting
4+
from tasks.schema.check_codegen_drift import check_codegen_drift
45
from tasks.schema.generate import codegen, compress, generate, produce_embedded, produce_jsonschema
56
from tasks.schema.lint import lint as lint_task
67
from tasks.schema.locate import locate as locate_task
78
from tasks.schema.template import template, template_all
89

910
collection = Collection()
1011
collection.add_task(add_setting)
12+
collection.add_task(check_codegen_drift)
1113
collection.add_task(codegen)
1214
collection.add_task(generate)
1315
collection.add_task(lint_task)

0 commit comments

Comments
 (0)