diff --git a/pkg/util/docker/BUILD.bazel b/pkg/util/docker/BUILD.bazel index d87fc2bf855b..5e64f8576be0 100644 --- a/pkg/util/docker/BUILD.bazel +++ b/pkg/util/docker/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "metadata.go", "metadata_no_docker.go", "rancher.go", + "safe_info.go", "storage.go", "util_common.go", "util_docker.go", @@ -51,6 +52,7 @@ go_test( "event_stream_test.go", "host_tags_test.go", "rancher_test.go", + "safe_info_test.go", "storage_test.go", "util_docker_test.go", ], @@ -59,7 +61,6 @@ go_test( deps = [ "//comp/core/workloadfilter/fx-mock", "//pkg/config/mock", - "//pkg/util/docker/fake", "@com_github_moby_moby_api//types/container", "@com_github_moby_moby_api//types/events", "@com_github_moby_moby_api//types/swarm", diff --git a/pkg/util/docker/docker_util.go b/pkg/util/docker/docker_util.go index 80ca211459f4..3c95f5245f88 100644 --- a/pkg/util/docker/docker_util.go +++ b/pkg/util/docker/docker_util.go @@ -84,10 +84,11 @@ func ConnectToDocker(ctx context.Context) (*client.Client, error) { if err != nil { return nil, err } - // Looks like docker is not actually doing a call to server when `NewClient` is called - // Forcing it to verify server availability by calling Info() - _, err = cli.Info(ctx, client.InfoOptions{}) - if err != nil { + // client.New does not actually contact the daemon. Force a round-trip + // to verify availability. safeInfo tolerates daemons that emit invalid + // CIDRs in /info's DefaultAddressPools, which would otherwise fail the + // strict netip.Prefix decoding introduced by the moby v29 client. + if _, err := safeInfo(ctx, cli); err != nil { return nil, err } @@ -182,11 +183,11 @@ func (d *DockerUtil) RawContainerListWithFilter(ctx context.Context, options cli func (d *DockerUtil) GetHostname(ctx context.Context) (string, error) { ctx, cancel := context.WithTimeout(ctx, d.queryTimeout) defer cancel() - result, err := d.cli.Info(ctx, client.InfoOptions{}) + info, err := safeInfo(ctx, d.cli) if err != nil { return "", fmt.Errorf("unable to get Docker info: %s", err) } - return result.Info.Name, nil + return info.Name, nil } // GetStorageStats returns the docker global storage stats if available @@ -194,11 +195,11 @@ func (d *DockerUtil) GetHostname(ctx context.Context) (string, error) { func (d *DockerUtil) GetStorageStats(ctx context.Context) ([]*StorageStats, error) { ctx, cancel := context.WithTimeout(ctx, d.queryTimeout) defer cancel() - result, err := d.cli.Info(ctx, client.InfoOptions{}) + info, err := safeInfo(ctx, d.cli) if err != nil { return []*StorageStats{}, fmt.Errorf("unable to get Docker info: %s", err) } - return parseStorageStatsFromInfo(result.Info) + return parseStorageStatsFromInfo(info) } func isImageShaOrRepoDigest(image string) bool { diff --git a/pkg/util/docker/host_tags.go b/pkg/util/docker/host_tags.go index 805d230237e8..4f753f837faf 100644 --- a/pkg/util/docker/host_tags.go +++ b/pkg/util/docker/host_tags.go @@ -13,7 +13,7 @@ import ( "time" "github.com/moby/moby/api/types/swarm" - "github.com/moby/moby/client" + "github.com/moby/moby/api/types/system" ) // GetTags returns tags that are automatically added to metrics and events on a @@ -25,24 +25,22 @@ func GetTags(ctx context.Context) ([]string, error) { } ctx, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() - return getTags(ctx, du.cli) + info, err := safeInfo(ctx, du.cli) + if err != nil { + return []string{}, err + } + return buildSwarmTags(info), nil } -func getTags(ctx context.Context, c client.SystemAPIClient) ([]string, error) { - tags := []string{} - result, err := c.Info(ctx, client.InfoOptions{}) - if err != nil { - return tags, err +// buildSwarmTags derives the docker swarm-related host tags from a daemon +// /info response. +func buildSwarmTags(info system.Info) []string { + if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive { + return []string{} } - switch result.Info.Swarm.LocalNodeState { - case swarm.LocalNodeStateActive: - nodeRole := swarm.NodeRoleWorker - if result.Info.Swarm.ControlAvailable { - nodeRole = swarm.NodeRoleManager - } - tags = append(tags, fmt.Sprintf("docker_swarm_node_role:%s", nodeRole)) - default: - break + nodeRole := swarm.NodeRoleWorker + if info.Swarm.ControlAvailable { + nodeRole = swarm.NodeRoleManager } - return tags, nil + return []string{fmt.Sprintf("docker_swarm_node_role:%s", nodeRole)} } diff --git a/pkg/util/docker/host_tags_test.go b/pkg/util/docker/host_tags_test.go index 0d3e3056776c..0c32437ba394 100644 --- a/pkg/util/docker/host_tags_test.go +++ b/pkg/util/docker/host_tags_test.go @@ -8,62 +8,45 @@ package docker import ( - "context" "testing" "github.com/moby/moby/api/types/swarm" "github.com/moby/moby/api/types/system" - "github.com/moby/moby/client" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/DataDog/datadog-agent/pkg/util/docker/fake" ) -func TestGetTags(t *testing.T) { +func TestBuildSwarmTags(t *testing.T) { tests := []struct { - desc string - client client.SystemAPIClient - tags []string + desc string + info system.Info + tags []string }{ { "manager node with swarm active", - &fake.SystemAPIClient{ - InfoFunc: func() (system.Info, error) { - return system.Info{ - Swarm: swarm.Info{ - LocalNodeState: swarm.LocalNodeStateActive, - ControlAvailable: true, - }, - }, nil + system.Info{ + Swarm: swarm.Info{ + LocalNodeState: swarm.LocalNodeStateActive, + ControlAvailable: true, }, }, []string{"docker_swarm_node_role:manager"}, }, { "worker node with swarm active", - &fake.SystemAPIClient{ - InfoFunc: func() (system.Info, error) { - return system.Info{ - Swarm: swarm.Info{ - LocalNodeState: swarm.LocalNodeStateActive, - ControlAvailable: false, - }, - }, nil + system.Info{ + Swarm: swarm.Info{ + LocalNodeState: swarm.LocalNodeStateActive, + ControlAvailable: false, }, }, []string{"docker_swarm_node_role:worker"}, }, { "swarm inactive", - &fake.SystemAPIClient{ - InfoFunc: func() (system.Info, error) { - return system.Info{ - Swarm: swarm.Info{ - LocalNodeState: swarm.LocalNodeStatePending, - ControlAvailable: true, - }, - }, nil + system.Info{ + Swarm: swarm.Info{ + LocalNodeState: swarm.LocalNodeStatePending, + ControlAvailable: true, }, }, []string{}, @@ -71,10 +54,7 @@ func TestGetTags(t *testing.T) { } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { - ctx := context.TODO() - tags, err := getTags(ctx, tt.client) - require.NoError(t, err) - assert.Equal(t, tt.tags, tags) + assert.Equal(t, tt.tags, buildSwarmTags(tt.info)) }) } } diff --git a/pkg/util/docker/metadata.go b/pkg/util/docker/metadata.go index 09f3937ceccb..0925ee07d5ba 100644 --- a/pkg/util/docker/metadata.go +++ b/pkg/util/docker/metadata.go @@ -14,7 +14,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/moby/moby/api/types/swarm" - "github.com/moby/moby/client" ) // GetMetadata returns metadata about the docker runtime such as docker_version and if docker_swarm is enabled or not. @@ -32,18 +31,18 @@ func GetMetadata() (map[string]string, error) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - result, err := du.cli.Info(ctx, client.InfoOptions{}) + info, err := safeInfo(ctx, du.cli) if err != nil { return nil, err } dockerSwarm := "inactive" - if result.Info.Swarm.LocalNodeState == swarm.LocalNodeStateActive { + if info.Swarm.LocalNodeState == swarm.LocalNodeStateActive { dockerSwarm = "active" } return map[string]string{ - "docker_version": result.Info.ServerVersion, + "docker_version": info.ServerVersion, "docker_swarm": dockerSwarm, }, nil } diff --git a/pkg/util/docker/safe_info.go b/pkg/util/docker/safe_info.go new file mode 100644 index 000000000000..1e16da5ee69e --- /dev/null +++ b/pkg/util/docker/safe_info.go @@ -0,0 +1,125 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build docker + +package docker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "path" + "strings" + + "github.com/moby/moby/api/types/system" + "github.com/moby/moby/client" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// safeInfo returns the Docker daemon's /info response, working around daemons +// that emit invalid CIDRs in DefaultAddressPools[].Base. The moby v29 SDK +// decodes Base into a netip.Prefix, whose UnmarshalText is strict and rejects +// such values, which would fail the entire /info JSON decode and break every +// caller (init probe, hostname provider, host tags, host metadata, storage +// stats). +// +// Strategy: try the SDK's Info() first. If it fails with the SDK's JSON-decode +// wrapper, retry with a raw HTTP request that decodes into a mirror struct +// where DefaultAddressPools is captured as json.RawMessage, shadowing the +// strict field and letting the rest of /info parse normally. +func safeInfo(ctx context.Context, cli *client.Client) (system.Info, error) { + result, err := cli.Info(ctx, client.InfoOptions{}) + if err == nil { + return result.Info, nil + } + + // The moby client wraps JSON-decode failures of /info with this prefix + // (see github.com/moby/moby/client/system_info.go). Network, HTTP-status + // and other connection errors do not, and the tolerant fallback would not + // help in those cases — propagate the original error. + if !strings.Contains(err.Error(), "Error reading remote info") { + return system.Info{}, err + } + + log.Debugf("Docker /info decode failed (%v); retrying with tolerant decoder", err) + info, fallbackErr := tolerantInfo(ctx, cli) + if fallbackErr != nil { + return system.Info{}, errors.Join(err, fmt.Errorf("tolerant /info fallback: %w", fallbackErr)) + } + return info, nil +} + +// tolerantInfo reissues GET /info through the moby client's dialer and decodes +// the response into a struct that shadows DefaultAddressPools with a +// json.RawMessage. This bypasses the strict netip.Prefix decoding of the typed +// field while leaving the rest of system.Info populated. +func tolerantInfo(ctx context.Context, cli *client.Client) (system.Info, error) { + httpClient := &http.Client{ + Transport: &http.Transport{ + // One-shot client: no benefit from keep-alive, and DisableKeepAlives + // ensures the dialed connection is closed when the response body + // is, so the unreferenced transport does not retain FDs. + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return cli.Dialer()(ctx) + }, + }, + } + + // The dialer takes care of reaching the daemon (unix, npipe, tcp, tcp+tls). + // For TCP daemons, preserve the configured host and base path so reverse + // proxies relying on Host-header or path routing reach the same backend + // as the SDK does. For unix/npipe, the SDK uses DummyHost — match it. + // Use the "http" scheme even for tls-fronted daemons: the dialer returns + // an already-TLS-encrypted connection, and the http transport writes plain + // HTTP bytes over it. + reqHost := client.DummyHost + basePath := "" + if hostURL, err := client.ParseHostURL(cli.DaemonHost()); err == nil { + if hostURL.Scheme == "tcp" { + reqHost = hostURL.Host + } + basePath = hostURL.Path + } + // Match the SDK's path construction so reverse proxies routing on + // /vX.Y/info don't reject the fallback (see moby client.getAPIPath). + apiPath := "/info" + if v := cli.ClientVersion(); v != "" { + apiPath = "/v" + strings.TrimPrefix(v, "v") + apiPath + } + url := "http://" + reqHost + path.Join("/", basePath, apiPath) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return system.Info{}, err + } + + resp, err := httpClient.Do(req) + if err != nil { + return system.Info{}, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return system.Info{}, fmt.Errorf("unexpected status %d from /info", resp.StatusCode) + } + + // Outer DefaultAddressPools (json.RawMessage) shadows the promoted field + // from the embedded system.Info: encoding/json routes the JSON value to + // the less-nested field, leaving system.Info.DefaultAddressPools at its + // zero value. All other fields decode through the embedded struct. + var tolerant struct { + system.Info + DefaultAddressPools json.RawMessage `json:"DefaultAddressPools"` + } + if err := json.NewDecoder(resp.Body).Decode(&tolerant); err != nil { + return system.Info{}, fmt.Errorf("tolerant decode of /info failed: %w", err) + } + return tolerant.Info, nil +} diff --git a/pkg/util/docker/safe_info_test.go b/pkg/util/docker/safe_info_test.go new file mode 100644 index 000000000000..399f28847c7d --- /dev/null +++ b/pkg/util/docker/safe_info_test.go @@ -0,0 +1,115 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build docker + +package docker + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// dockerHostFromTestServer rewrites a httptest server URL ("http://127.0.0.1:PORT") +// to the form expected by moby's WithHost option ("tcp://127.0.0.1:PORT"). +func dockerHostFromTestServer(serverURL string) string { + return "tcp://" + strings.TrimPrefix(serverURL, "http://") +} + +func newTestDockerClient(t *testing.T, serverURL string) *client.Client { + t.Helper() + cli, err := client.New(client.WithHost(dockerHostFromTestServer(serverURL))) + require.NoError(t, err) + return cli +} + +// fakeDockerDaemon is a minimal stand-in for the moby daemon that answers +// /_ping (used by the client for version negotiation) and any /info path +// (possibly version-prefixed, e.g. /v1.54/info) with the supplied body. +func fakeDockerDaemon(t *testing.T, infoBody string, infoStatus int) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/_ping": + w.Header().Set("API-Version", "1.54") + w.Header().Set("OSType", "linux") + w.WriteHeader(http.StatusOK) + case strings.HasSuffix(r.URL.Path, "/info"): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(infoStatus) + if infoStatus == http.StatusOK { + fmt.Fprint(w, infoBody) + } + default: + t.Errorf("unexpected request path %q", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) +} + +func TestSafeInfo_HappyPath(t *testing.T) { + body := `{ + "Name": "test-daemon", + "ServerVersion": "29.0.1", + "Swarm": {"LocalNodeState": "active", "ControlAvailable": true} + }` + server := fakeDockerDaemon(t, body, http.StatusOK) + defer server.Close() + + cli := newTestDockerClient(t, server.URL) + info, err := safeInfo(t.Context(), cli) + require.NoError(t, err) + assert.Equal(t, "test-daemon", info.Name) + assert.Equal(t, "29.0.1", info.ServerVersion) + assert.Equal(t, swarm.LocalNodeStateActive, info.Swarm.LocalNodeState) + assert.True(t, info.Swarm.ControlAvailable) +} + +func TestSafeInfo_FallbackOnInvalidPrefix(t *testing.T) { + // Reproduces the failure mode reported in incident #54830: the daemon emits + // "invalid Prefix" (netip.Prefix.String() of an invalid prefix) for the + // Base field of a DefaultAddressPools entry, which trips moby v29's strict + // netip.Prefix decoding and fails the entire /info JSON decode. + body := `{ + "Name": "broken-daemon", + "ServerVersion": "28.5.0", + "Swarm": {"LocalNodeState": "inactive", "ControlAvailable": false}, + "DefaultAddressPools": [ + {"Base": "invalid Prefix", "Size": 0} + ] + }` + server := fakeDockerDaemon(t, body, http.StatusOK) + defer server.Close() + + cli := newTestDockerClient(t, server.URL) + info, err := safeInfo(t.Context(), cli) + require.NoError(t, err) + assert.Equal(t, "broken-daemon", info.Name) + assert.Equal(t, "28.5.0", info.ServerVersion) + assert.Equal(t, swarm.LocalNodeStateInactive, info.Swarm.LocalNodeState) + // The problematic field is intentionally dropped by the tolerant decoder. + assert.Empty(t, info.DefaultAddressPools) +} + +func TestSafeInfo_PropagatesNonDecodeErrors(t *testing.T) { + // The fallback only kicks in for JSON-decode failures of /info. A daemon + // returning a non-2xx status must propagate the original SDK error without + // engaging the fallback. + server := fakeDockerDaemon(t, "", http.StatusInternalServerError) + defer server.Close() + + cli := newTestDockerClient(t, server.URL) + _, err := safeInfo(t.Context(), cli) + require.Error(t, err) + assert.NotContains(t, err.Error(), "tolerant /info fallback") +} diff --git a/releasenotes/notes/docker-tolerant-info-decode-22bb5222c3b84324.yaml b/releasenotes/notes/docker-tolerant-info-decode-22bb5222c3b84324.yaml new file mode 100644 index 000000000000..d8cf80167cda --- /dev/null +++ b/releasenotes/notes/docker-tolerant-info-decode-22bb5222c3b84324.yaml @@ -0,0 +1,21 @@ +--- +fixes: + - | + Fix the Agent's Docker integration against Docker daemons that return + malformed values in their ``/info`` response. The failure was visible in + Agent logs as:: + + Docker init error: temporary failure in dockerutil, will retry later: + Error reading remote info: netip.ParsePrefix("invalid Prefix"): no '/' + + When triggered, it prevented the Docker integration from initializing, + which cascaded into: + + * missing container and image tags on metrics, traces and logs collected + from Docker containers, + * missing ``docker_version`` and ``docker_swarm`` entries in host + metadata, + * missing ``docker_swarm_node_role`` host tag on Docker Swarm nodes, + * in containerized deployments without an explicit ``DD_HOSTNAME``, the + Agent could refuse to start because the Docker hostname provider could + no longer determine a hostname.