Skip to content

Commit a06b857

Browse files
committed
Introduce a CI job that test the config generated code
1 parent 117e85b commit a06b857

9 files changed

Lines changed: 410 additions & 1 deletion

File tree

.gitlab/JOBOWNERS

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ build_windows_container_entrypoint @DataDog/windows-products
5757
generate_config_schema-linux @DataDog/fleet-automation
5858
generate_config_schema-macos @DataDog/fleet-automation
5959
generate_config_schema-windows @DataDog/fleet-automation
60-
# Cross-compiled binary builds for validation purpose
60+
check_config_codegen_drift-linux @DataDog/fleet-automation
61+
check_config_codegen_drift-macos @DataDog/fleet-automation
62+
check_config_codegen_drift-windows @DataDog/fleet-automation
6163
cross_build_agent-binary_aix_ppc64 @DataDog/agent-build
6264
cross_build_trace_agent-binary_aix_ppc64 @DataDog/agent-build
6365

.gitlab/build/binary_build/schema_generation.yml

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,90 @@ generate_config_schema-windows:
9898
expire_in: 2 weeks
9999
paths:
100100
- $CI_PROJECT_DIR/pkg/config/schema/yaml/*.yaml
101+
102+
# Verify that regenerating the config settings code (`dda inv schema.codegen`)
103+
# does not change the Agent's runtime configuration defaults. See the
104+
# schema.check-codegen-drift invoke task for the full flow.
105+
.check_config_codegen_drift:
106+
stage: binary_build
107+
rules:
108+
- !reference [.except_mergequeue]
109+
- !reference [.on_main_or_release_branch]
110+
- !reference [.on_schema_paths_changed]
111+
112+
check_config_codegen_drift-linux:
113+
extends:
114+
- .check_config_codegen_drift
115+
- .bazel:defs:cache:progressive
116+
variables:
117+
KUBERNETES_CPU_REQUEST: 16
118+
KUBERNETES_MEMORY_REQUEST: 16Gi
119+
KUBERNETES_MEMORY_LIMIT: 16Gi
120+
TEST_OUTPUT_FILE: test_output
121+
image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
122+
tags: ["arch:amd64", "specific:true"]
123+
needs: ["go_deps"]
124+
before_script:
125+
- !reference [.retrieve_linux_go_deps]
126+
script:
127+
- dda inv -- schema.check-codegen-drift --output-dir $CI_PROJECT_DIR/config_codegen_drift
128+
artifacts:
129+
when: always
130+
expire_in: 2 weeks
131+
paths:
132+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
133+
134+
check_config_codegen_drift-macos:
135+
extends:
136+
- .check_config_codegen_drift
137+
- .bazel:defs:cache:macos
138+
- .macos_gitlab
139+
variables:
140+
KUBERNETES_CPU_REQUEST: 16
141+
KUBERNETES_MEMORY_REQUEST: 16Gi
142+
KUBERNETES_MEMORY_LIMIT: 16Gi
143+
TEST_OUTPUT_FILE: test_output
144+
tags: ["macos:sonoma-amd64", "specific:true"]
145+
needs: ["go_deps", "go_tools_deps"]
146+
script:
147+
- dda inv -- schema.check-codegen-drift --output-dir $CI_PROJECT_DIR/config_codegen_drift
148+
artifacts:
149+
when: always
150+
expire_in: 2 weeks
151+
paths:
152+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
153+
154+
check_config_codegen_drift-windows:
155+
extends:
156+
- .check_config_codegen_drift
157+
- .bazel:defs:cache:windows
158+
- .windows_docker_default
159+
needs: ["go_deps", "go_tools_deps"]
160+
variables:
161+
ARCH: "x64"
162+
script:
163+
- $ErrorActionPreference = "Stop"
164+
- !reference [.sanitize_goproxy_windows]
165+
- >
166+
.\tools\ci\docker-run-with-bazel-cache.ps1
167+
-m 24576M
168+
-v "$(Get-Location):c:\mnt"
169+
-e GITLAB_CI
170+
-e CI_JOB_ID
171+
-e CI_PIPELINE_ID
172+
-e CI_PROJECT_NAME
173+
-e AWS_NETWORKING=true
174+
-e GOMODCACHE="c:\modcache"
175+
-e GOPROXY
176+
-e GONOSUMDB
177+
-e PIP_INDEX_URL
178+
-e DDA_FEATURE_FLAGS_CI_SSM_KEY_WINDOWS
179+
-e CI_IDENTITIES_GITLAB_ID_TOKEN
180+
${WINBUILDIMAGE}
181+
powershell.exe -c "c:\mnt\tasks\winbuildscripts\Check-ConfigCodegenDrift.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1"
182+
- If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
183+
artifacts:
184+
when: always
185+
expire_in: 2 weeks
186+
paths:
187+
- $CI_PROJECT_DIR/config_codegen_drift/*.json
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
load("@rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "dumpconfig",
5+
srcs = ["command.go"],
6+
importpath = "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dumpconfig",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//cmd/agent/command",
10+
"//pkg/config/model",
11+
"//pkg/config/setup",
12+
"@com_github_spf13_cobra//:cobra",
13+
],
14+
)
15+
16+
go_test(
17+
name = "dumpconfig_test",
18+
srcs = ["command_test.go"],
19+
embed = [":dumpconfig"],
20+
gotags = ["test"],
21+
deps = [
22+
"//cmd/agent/command",
23+
"@com_github_stretchr_testify//assert",
24+
"@com_github_stretchr_testify//require",
25+
],
26+
)
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
// Package dumpconfig implements 'agent dumpconfig'.
7+
//
8+
// It initializes the configuration defaults (without reading any datadog.yaml
9+
// or system-probe.yaml file) and prints the resulting runtime configuration as
10+
// JSON. It is used by CI to verify that regenerating the config settings code
11+
// (`dda inv schema.codegen`) does not change the Agent's runtime defaults.
12+
package dumpconfig
13+
14+
import (
15+
"encoding/json"
16+
"fmt"
17+
"time"
18+
19+
"github.com/spf13/cobra"
20+
21+
"github.com/DataDog/datadog-agent/cmd/agent/command"
22+
"github.com/DataDog/datadog-agent/pkg/config/model"
23+
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
24+
)
25+
26+
// cliParams are the command-line arguments for this subcommand
27+
type cliParams struct {
28+
*command.GlobalParams
29+
Target string
30+
}
31+
32+
// Commands returns a slice of subcommands for the 'agent' command.
33+
func Commands(globalParams *command.GlobalParams) []*cobra.Command {
34+
cliParams := &cliParams{
35+
GlobalParams: globalParams,
36+
}
37+
38+
dumpConfigCommand := &cobra.Command{
39+
Use: "dumpconfig",
40+
Short: "Dump the runtime configuration defaults as JSON",
41+
Long: ``,
42+
Hidden: true,
43+
RunE: func(_ *cobra.Command, _ []string) error {
44+
return run(cliParams)
45+
},
46+
}
47+
dumpConfigCommand.Flags().StringVar(&cliParams.Target, "target", "", "config to dump: core or system-probe")
48+
49+
return []*cobra.Command{dumpConfigCommand}
50+
}
51+
52+
func run(cliParams *cliParams) error {
53+
// Initialize the global config objects with their defaults only. No
54+
// datadog.yaml or system-probe.yaml file is loaded, so what we dump is the
55+
// pure set of runtime defaults.
56+
pkgconfigsetup.InitConfigObjects()
57+
58+
var cfg model.Config
59+
switch cliParams.Target {
60+
case "core":
61+
cfg = pkgconfigsetup.Datadog()
62+
case "system-probe":
63+
cfg = pkgconfigsetup.SystemProbe()
64+
default:
65+
return fmt.Errorf("unknown target '%s', valid ones are 'core' or 'system-probe'", cliParams.Target)
66+
}
67+
68+
// json.Marshal sorts map keys, so the output is deterministic across runs.
69+
data, err := json.MarshalIndent(durationsToString(cfg.AllSettings()), "", " ")
70+
if err != nil {
71+
return err
72+
}
73+
fmt.Println(string(data))
74+
return nil
75+
}
76+
77+
// durationsToString recursively walks a value decoded from the config and
78+
// replaces every time.Duration with its string representation (e.g. "10s"),
79+
// so durations are dumped as human-readable strings instead of raw
80+
// nanosecond integers.
81+
func durationsToString(v interface{}) interface{} {
82+
switch val := v.(type) {
83+
case time.Duration:
84+
return val.String()
85+
case map[string]interface{}:
86+
for k, elem := range val {
87+
val[k] = durationsToString(elem)
88+
}
89+
return val
90+
case []interface{}:
91+
for i, elem := range val {
92+
val[i] = durationsToString(elem)
93+
}
94+
return val
95+
default:
96+
return v
97+
}
98+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
package dumpconfig
7+
8+
import (
9+
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
14+
"github.com/DataDog/datadog-agent/cmd/agent/command"
15+
)
16+
17+
func TestRun(t *testing.T) {
18+
for _, target := range []string{"core", "system-probe"} {
19+
t.Run(target, func(t *testing.T) {
20+
require.NoError(t, run(&cliParams{GlobalParams: &command.GlobalParams{}, Target: target}))
21+
})
22+
}
23+
}
24+
25+
func TestRunUnknownTarget(t *testing.T) {
26+
err := run(&cliParams{GlobalParams: &command.GlobalParams{}, Target: "unknown"})
27+
assert.ErrorContains(t, err, "unknown target")
28+
}

cmd/agent/subcommands/subcommands.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
cmddogstatsdcapture "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdcapture"
2121
cmddogstatsdreplay "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdreplay"
2222
cmddogstatsdstats "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dogstatsdstats"
23+
cmddumpconfig "github.com/DataDog/datadog-agent/cmd/agent/subcommands/dumpconfig"
2324
cmdexperimental "github.com/DataDog/datadog-agent/cmd/agent/subcommands/experimental"
2425
cmdflare "github.com/DataDog/datadog-agent/cmd/agent/subcommands/flare"
2526
cmdhealth "github.com/DataDog/datadog-agent/cmd/agent/subcommands/health"
@@ -60,6 +61,7 @@ func AgentSubcommands() []command.SubcommandFactory {
6061
cmddogstatsdcapture.Commands,
6162
cmddogstatsdreplay.Commands,
6263
cmddogstatsdstats.Commands,
64+
cmddumpconfig.Commands,
6365
cmdflare.Commands,
6466
cmdhealth.Commands,
6567
cmdhostname.Commands,

tasks/schema/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
from invoke.collection import Collection
22

33
from tasks.schema.add_setting import add_setting
4+
from tasks.schema.check_codegen_drift import check_codegen_drift
45
from tasks.schema.generate import codegen, compress, generate, produce_embedded, produce_jsonschema
56
from tasks.schema.lint import lint as lint_task
67
from tasks.schema.locate import locate as locate_task
78
from tasks.schema.template import template, template_all
89

910
collection = Collection()
1011
collection.add_task(add_setting)
12+
collection.add_task(check_codegen_drift)
1113
collection.add_task(codegen)
1214
collection.add_task(generate)
1315
collection.add_task(lint_task)

0 commit comments

Comments
 (0)