Don't call Marshal in unmarshalerEmbeddedStructsHookFunc (#14213)

## Context

`confmap` uses a `mapstructure` hook to call the `Unmarshal` method on
types which define it as part of the unmarshaling process, but
`mapstructure` hooks are not called for squashed fields, so we have a
separate hook which manually inspects all structs for squashed fields
that implement `Unmarshaler`.

This second hook currently returns a modified version of the raw YAML
config, to be used by `mapstructure` to finish unmarshaling the full
struct. This is a problem, because an `Unmarshal` method can set the
state of the struct in arbitrary ways, which we want to preserve without
`mapstructure` overwriting fields. (See `TestEmbeddedUnmarshaler`)

The way this is currently worked around is by marshaling the embedded
struct into a map, and then copying that map back into the raw config,
in the hope that nothing will change when the raw config is mapped back
into the struct.

However, this assumption breaks when dealing with a
`configopaque.String` inside an embedded `Unmarshaler`. That type has
the very deliberate behavior of returning `[REDACTED]` when marshaled
instead of its true contents, meaning marshaling then unmarshaling does
not round trip.

## Description

This PR attempts to fix this issue and make embedded `Unmarshaler`s more
reliable by modifying the hook so that it returns a fully unmarshaled
struct, leading `mapstructure` to do nothing. (This is what the basic
`Unmarshaler` hook does to avoid "interference".)

This means the hook needs to somehow unmarshal the non-`Unmarshaler`
subfields manually, without affecting the input or output of
`Unmarshal`. I did this by constructing a custom "partial" struct using
`reflect.StructOf` which only contains the non-`Unmarshaler` fields,
copying the initial values into it, calling `Decode` on it, then copying
the output values back into the original struct.

In a previous version of this PR, I kept most of the hook intact, but
changed the logic so that we clear fields from the raw config when they
might conflict with the embedded struct (instead of setting those fields
to the embedded struct's current value). However, this has the side
effect of breaking unmarshaling of structs with multiple
identically-named fields, of which there is [at least one in
contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/320ef1ca9bb4593b498eb3d849659c55d6a7eccc/receiver/dockerstatsreceiver/config.go#L17)
(both `docker.Config` and `scraperhelper.ControllerConfig` have a
`Timeout` field, and one of them implements `Unmarshaler`).

---------

Co-authored-by: Pablo Baeyens <pablo.baeyens@datadoghq.com>

Author: jade-guiton-dd

.chloggen/no-marshal-in-unmarshal-hook.yaml

@@ -0,0 +1,26 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: pkg/confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Ensure that embedded structs are not overwritten after Unmarshal is called
+
+# One or more tracking issues or pull requests related to the change
+issues: [14213]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This allows embedding structs which implement Unmarshal and contain a configopaque.String.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

confmap/internal/confmap_test.go

@@ -4,6 +4,7 @@
 package internal
 
 import (
+	"encoding"
 	"errors"
 	"math"
 	"os"
@@ -559,6 +560,53 @@ func TestEmbeddedMarshalerError(t *testing.T) {
 	assert.EqualError(t, cfgMap.Unmarshal(tc), "error running encode hook: marshaling error")
 }
 
+// stringOpaque is similar to configopaque.String, in that
+// marshaling then unmarshaling it changes its value.
+type stringOpaque string
+
+var _ encoding.TextMarshaler = stringOpaque("")
+
+func (s stringOpaque) MarshalText() (text []byte, err error) {
+	return []byte("opaque"), nil
+}
+
+type squashedConfigOpaque struct {
+	Value  stringOpaque `mapstructure:"value"`
+	secret bool
+}
+
+var _ Unmarshaler = (*squashedConfigOpaque)(nil)
+
+func (ecwrt *squashedConfigOpaque) Unmarshal(conf *Conf) error {
+	if err := conf.Unmarshal(ecwrt); err != nil {
+		return err
+	}
+	ecwrt.secret = true
+	return nil
+}
+
+type testConfigOpaque struct {
+	// Don't embed, otherwise testConfigOpaque will implement Unmarshaler,
+	// and unmarshalerHookFunc will be called instead of unmarshalerEmbeddedStructsHookFunc.
+	Squashed squashedConfigOpaque `mapstructure:",squash"`
+}
+
+// Regression test: the hook processing embedded marshalers previously made the assumption that
+// marshaling a struct then unmarshaling the resulting map back into the struct
+// is a no-op, which is not true for configopaque.String.
+func TestEmbeddedMarshalerWithoutRoundtrip(t *testing.T) {
+	cfgMap := NewFromStringMap(map[string]any{
+		"value": "hello",
+	})
+
+	tc := &testConfigOpaque{}
+	require.NoError(t, cfgMap.Unmarshal(tc))
+	// Check that "hello" hasn't been replaced by "opaque"
+	assert.Equal(t, "hello", string(tc.Squashed.Value))
+	// Check that the inner Unmarshal was called
+	assert.True(t, tc.Squashed.secret)
+}
+
 type B struct {
 	String string `mapstructure:"string"`
 }

confmap/internal/decoder.go

@@ -7,7 +7,6 @@ import (
 	"encoding"
 	"errors"
 	"fmt"
-	"maps"
 	"reflect"
 	"slices"
 	"strings"
@@ -57,7 +56,7 @@ func Decode(input, result any, settings UnmarshalOptions, skipTopLevelUnmarshale
 			unmarshalerHookFunc(result, skipTopLevelUnmarshaler),
 			// after the main unmarshaler hook is called,
 			// we unmarshal the embedded structs if present to merge with the result:
-			unmarshalerEmbeddedStructsHookFunc(),
+			unmarshalerEmbeddedStructsHookFunc(settings),
 			zeroSliceAndMapHookFunc(),
 		),
 	}
@@ -189,7 +188,7 @@ func mapKeyStringToMapKeyTextUnmarshalerHookFunc() mapstructure.DecodeHookFuncTy
 
 // unmarshalerEmbeddedStructsHookFunc provides a mechanism for embedded structs to define their own unmarshal logic,
 // by implementing the Unmarshaler interface.
-func unmarshalerEmbeddedStructsHookFunc() mapstructure.DecodeHookFuncValue {
+func unmarshalerEmbeddedStructsHookFunc(settings UnmarshalOptions) mapstructure.DecodeHookFuncValue {
 	return safeWrapDecodeHookFunc(func(from, to reflect.Value) (any, error) {
 		if to.Type().Kind() != reflect.Struct {
 			return from.Interface(), nil
@@ -198,32 +197,72 @@ func unmarshalerEmbeddedStructsHookFunc() mapstructure.DecodeHookFuncValue {
 		if !ok {
 			return from.Interface(), nil
 		}
+
+		// First call Unmarshaler on squashed embedded fields, if necessary.
+		var squashedUnmarshalers []int
 		for i := 0; i < to.Type().NumField(); i++ {
-			// embedded structs passed in via `squash` cannot be pointers. We just check if they are structs:
 			f := to.Type().Field(i)
-			if f.IsExported() && slices.Contains(strings.Split(f.Tag.Get(MapstructureTag), ","), "squash") {
-				if unmarshaler, ok := to.Field(i).Addr().Interface().(Unmarshaler); ok {
-					c := NewFromStringMap(fromAsMap)
-					c.skipTopLevelUnmarshaler = true
-					if err := unmarshaler.Unmarshal(c); err != nil {
-						return nil, err
-					}
-					// the struct we receive from this unmarshaling only contains fields related to the embedded struct.
-					// we merge this partially unmarshaled struct with the rest of the result.
-					// note we already unmarshaled the main struct earlier, and therefore merge with it.
-					conf := New()
-					if err := conf.Marshal(unmarshaler); err != nil {
-						return nil, err
-					}
-					resultMap := conf.ToStringMap()
-					if fromAsMap == nil && len(resultMap) > 0 {
-						fromAsMap = make(map[string]any, len(resultMap))
-					}
-					maps.Copy(fromAsMap, resultMap)
-				}
+			if !f.IsExported() {
+				continue
+			}
+			tagParts := strings.Split(f.Tag.Get(MapstructureTag), ",")
+			if !slices.Contains(tagParts[1:], "squash") {
+				continue
+			}
+			unmarshaler, ok := to.Field(i).Addr().Interface().(Unmarshaler)
+			if !ok {
+				continue
 			}
+			c := NewFromStringMap(fromAsMap)
+			c.skipTopLevelUnmarshaler = true
+			if err := unmarshaler.Unmarshal(c); err != nil {
+				return nil, err
+			}
+			squashedUnmarshalers = append(squashedUnmarshalers, i)
+		}
+
+		// No squashed unmarshalers, we can let mapstructure do its job.
+		if len(squashedUnmarshalers) == 0 {
+			return fromAsMap, nil
 		}
-		return fromAsMap, nil
+
+		// We need to unmarshal into all other fields without overwriting the output of the Unmarshal calls.
+		// To do that, create a custom "partial" struct containing only the non-squashed fields.
+		var fields []reflect.StructField
+		var fieldValues []reflect.Value
+		for i := 0; i < to.Type().NumField(); i++ {
+			f := to.Type().Field(i)
+			if !f.IsExported() {
+				continue
+			}
+			if slices.Contains(squashedUnmarshalers, i) {
+				continue
+			}
+			fields = append(fields, f)
+			fieldValues = append(fieldValues, to.Field(i))
+		}
+		restType := reflect.StructOf(fields)
+		restValue := reflect.New(restType)
+
+		// Copy initial values into partial struct.
+		for i, fieldValue := range fieldValues {
+			restValue.Elem().Field(i).Set(fieldValue)
+		}
+
+		// Decode into the partial struct.
+		// This performs a recursive call into this hook, which will be handled by the "no squashed unmarshalers" case above.
+		// We need to set `IgnoreUnused` to avoid errors from the map containing fields only present in the full struct.
+		settings.IgnoreUnused = true
+		if err := Decode(fromAsMap, restValue.Interface(), settings, true); err != nil {
+			return nil, err
+		}
+
+		// Copy decoding results back to the original struct.
+		for i, fieldValue := range fieldValues {
+			fieldValue.Set(restValue.Elem().Field(i))
+		}
+
+		return to, nil
 	})
 }