feat: unknown field caching when decoding

Allow caching of unknown fields during PopulateStructFrom(CBOR|JSON).
The cache is then used during SerializeStructTo(CBOR|JSON). This ensures
that any map entries in the source data that do not correspond to target
struct fields are preserved across decode/encode cycle.

This feature is enabled by adding a field to a struct tagged with
`field-cache:""`. This field must be of type map[string]any. When
PopulateStructFrom* functions encounter input entries that do not
correspond to a field in the target struct, they will be added to the
field-cache map instead. Analogously, when SerializeStructTo* functions
see a field-cache map, they will add its entries to the output.

This scheme has some (hopefully, obvious) limitations:

- field-cache field's tag must also contain `cbor:"-" json:"-"` to make
  sure that the field itself will be ignored by serializers.
- if a struct is unmarshaled from JSON, any unknown field names
  must be "stringified" integers, otherwise it is impossible to obtain
  the corresponding CBOR code point mapping for the name.

Signed-off-by: Sergei Trofimov <[email protected]>

Author: setrofim

encoding/cbor.go

@@ -1,4 +1,4 @@
-// Copyright 2023-2024 Contributors to the Veraison project.
+// Copyright 2023-2025 Contributors to the Veraison project.
 // SPDX-License-Identifier: Apache-2.0
 
 package encoding
@@ -49,6 +49,13 @@ func doSerializeStructToCBOR(
 			continue
 		}
 
+		_, ok := typeField.Tag.Lookup("field-cache")
+		if ok {
+			if err := addCachedFieldsToMapCBOR(em, valField, rawMap); err != nil {
+				return err
+			}
+		}
+
 		tag, ok := typeField.Tag.Lookup("cbor")
 		if !ok {
 			continue
@@ -129,6 +136,7 @@ func doPopulateStructFromCBOR(
 	}
 
 	var embeds []embedded
+	var fieldCache reflect.Value
 
 	for i := 0; i < structVal.NumField(); i++ {
 		typeField := structType.Field(i)
@@ -138,6 +146,12 @@ func doPopulateStructFromCBOR(
 			continue
 		}
 
+		_, ok := typeField.Tag.Lookup("field-cache")
+		if ok {
+			fieldCache = valField
+			continue
+		}
+
 		tag, ok := typeField.Tag.Lookup("cbor")
 		if !ok {
 			continue
@@ -192,7 +206,9 @@ func doPopulateStructFromCBOR(
 		}
 	}
 
-	return nil
+	// Any remaining contents of rawMap will be added to the field cache,
+	// if current struct has one.
+	return updateFieldCacheCBOR(dm, fieldCache, rawMap)
 }
 
 // structFieldsCBOR is a specialized implementation of "OrderedMap", where the
@@ -414,3 +430,73 @@ func processAdditionalInfo(
 
 	return mapLen, rest, nil
 }
+
+func addCachedFieldsToMapCBOR(em cbor.EncMode, cacheField reflect.Value, rawMap *structFieldsCBOR) error {
+	if !isMapStringAny(cacheField) {
+		return errors.New("field-cache does not appear to be a map[string]any")
+	}
+
+	if !cacheField.IsValid() || cacheField.IsNil() {
+		// field cache was never set, so nothing to do
+		return nil
+	}
+
+	for _, key := range cacheField.MapKeys() {
+		keyText := key.String()
+		keyInt, err := strconv.Atoi(keyText)
+		if err != nil {
+			return fmt.Errorf(
+				"cached field name not an integer (cannot encode to CBOR): %s",
+				keyText,
+			)
+		}
+
+		data, err := em.Marshal(cacheField.MapIndex(key).Interface())
+		if err != nil {
+			return fmt.Errorf(
+				"error marshaling field-cache entry %q: %w",
+				keyText,
+				err,
+			)
+		}
+
+		if err := rawMap.Add(keyInt, cbor.RawMessage(data)); err != nil {
+			return fmt.Errorf(
+				"could not add field-cache entry %q to serialization map: %w",
+				keyText,
+				err,
+			)
+		}
+	}
+
+	return nil
+}
+
+func updateFieldCacheCBOR(dm cbor.DecMode, cacheField reflect.Value, rawMap *structFieldsCBOR) error {
+	if !cacheField.IsValid() {
+		// current struct does not have a field-cache field
+		return nil
+	}
+
+	if !isMapStringAny(cacheField) {
+		return errors.New("field-cache does not appear to be a map[string]any")
+	}
+
+	if cacheField.IsNil() {
+		cacheField.Set(reflect.MakeMap(cacheField.Type()))
+	}
+
+	for key, rawVal := range rawMap.Fields {
+		var val any
+		if err := dm.Unmarshal(rawVal, &val); err != nil {
+			return fmt.Errorf("could not unmarshal key %d: %w", key, err)
+		}
+
+		keyText := fmt.Sprint(key)
+		keyVal := reflect.ValueOf(keyText)
+		valVal := reflect.ValueOf(val)
+		cacheField.SetMapIndex(keyVal, valVal)
+	}
+
+	return nil
+}

encoding/embedded.go

@@ -1,4 +1,4 @@
-// Copyright 2024 Contributors to the Veraison project.
+// Copyright 2024-2025 Contributors to the Veraison project.
 // SPDX-License-Identifier: Apache-2.0
 package encoding
 
@@ -49,3 +49,21 @@ func collectEmbedded(
 
 	return false
 }
+
+// isMapStringAny returns true iff the provided value, v, is of type
+// map[string]any.
+func isMapStringAny(v reflect.Value) bool {
+	if v.Kind() != reflect.Map {
+		return false
+	}
+
+	if v.Type().Key().Kind() != reflect.String {
+		return false
+	}
+
+	if v.Type().Elem().Kind() != reflect.Interface {
+		return false
+	}
+
+	return true
+}

encoding/field_cache_test.go

@@ -0,0 +1,179 @@
+// Copyright 2025 Contributors to the Veraison project.
+// SPDX-License-Identifier: Apache-2.0
+package encoding
+
+import (
+	"testing"
+
+	cbor "github.com/fxamacker/cbor/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+// The following structs emulate the embedding pattern used for extensions
+
+type IEmbeddedValue any
+
+type Embedded struct {
+	IEmbeddedValue `json:"embedded,omitempty"`
+
+	FieldCache map[string]any `field-cache:"" cbor:"-" json:"-"`
+}
+
+type MyStruct struct {
+	Field0 string `cbor:"0,keyasint,omitempty" json:"field0,omitempty"`
+	Field1 int    `cbor:"1,keyasint,omitempty" json:"field1,omitempty"`
+
+	Embedded
+}
+
+type MyEmbed struct {
+	Foo string `cbor:"-1,keyasint,omitempty" json:"foo,omitempty"`
+}
+
+type EmbeddedNoCache struct {
+	IEmbeddedValue `json:"embedded,omitempty"`
+}
+
+type MyStructNoCache struct {
+	Field0 string `cbor:"0,keyasint,omitempty" json:"field0,omitempty"`
+	Field1 int    `cbor:"1,keyasint,omitempty" json:"field1,omitempty"`
+
+	EmbeddedNoCache
+}
+
+func mustInitEncMode() cbor.EncMode {
+	encOpt := cbor.EncOptions{
+		Sort:        cbor.SortCoreDeterministic,
+		IndefLength: cbor.IndefLengthForbidden,
+		TimeTag:     cbor.EncTagRequired,
+	}
+
+	em, err := encOpt.EncMode()
+	if err != nil {
+		panic(err)
+	}
+
+	return em
+}
+
+func mustInitDecMode() cbor.DecMode {
+	decOpt := cbor.DecOptions{
+		IndefLength: cbor.IndefLengthAllowed,
+	}
+
+	dm, err := decOpt.DecMode()
+	if err != nil {
+		panic(err)
+	}
+
+	return dm
+}
+
+func Test_preserve_unknown_embeds_CBOR(t *testing.T) {
+	// nolint: gocritic
+	data := []byte{
+		0xa3, // map(3)
+
+		0x00,       // key: 0
+		0x62,       // value: tstr(2)
+		0x66, 0x31, // "f1"
+
+		0x01, // key: 1
+		0x02, // value: 2
+
+		0x20,             // key: -1
+		0x63,             // value: tstr(3)
+		0x62, 0x61, 0x72, // "bar"
+	}
+
+	em := mustInitEncMode()
+	dm := mustInitDecMode()
+
+	// First, a sanity test to make sure that embedded data
+	// is preserved when there is a concrete struct to
+	// contain it.
+	embed := MyEmbed{}
+	myStruct := MyStruct{Embedded: Embedded{IEmbeddedValue: &embed, FieldCache: map[string]any{}}}
+
+	err := PopulateStructFromCBOR(dm, data, &myStruct)
+	assert.NoError(t, err)
+
+	outData, err := SerializeStructToCBOR(em, &myStruct)
+	assert.NoError(t, err)
+	assert.Equal(t, data, outData)
+
+	// Now, the same test with IEmbeddedValue not set. This simulates the
+	// case where extensions are present in the data but the struct needed
+	// to understand them has not been registered.
+	myStruct = MyStruct{}
+
+	err = PopulateStructFromCBOR(dm, data, &myStruct)
+	assert.NoError(t, err)
+
+	outData, err = SerializeStructToCBOR(em, &myStruct)
+	assert.NoError(t, err)
+	assert.Equal(t, data, outData)
+
+	// Make sure that, without caching, unknown values are simply ignored
+	// without causing errors.
+	noCache := MyStructNoCache{}
+	err = PopulateStructFromCBOR(dm, data, &noCache)
+	assert.NoError(t, err)
+
+	// nolint: gocritic
+	expectedNoEmbed := []byte{
+		0xa2, // map(2)
+
+		0x00,       // key: 0
+		0x62,       // value: tstr(2)
+		0x66, 0x31, // "f1"
+
+		0x01, // key: 1
+		0x02, // value: 2
+	}
+
+	outData, err = SerializeStructToCBOR(em, &noCache)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedNoEmbed, outData)
+}
+
+func Test_preserve_unknown_embeds_JSON(t *testing.T) {
+	data := []byte(`{"field0":"f1","field1":2,"foo":"bar"}`)
+
+	// First, a sanity test to make sure that embedded data
+	// is presevered when there is a concrete struct to
+	// contain it.
+	embed := MyEmbed{}
+	myStruct := MyStruct{Embedded: Embedded{IEmbeddedValue: &embed, FieldCache: map[string]any{}}}
+
+	err := PopulateStructFromJSON(data, &myStruct)
+	assert.NoError(t, err)
+
+	outData, err := SerializeStructToJSON(&myStruct)
+	assert.NoError(t, err)
+	assert.Equal(t, data, outData)
+
+	// Now, the same test with IEmbeddedValue not set. This simulates the
+	// case where extensions are present int the data but the struct needed
+	// to understand them has not been registered.
+	myStruct = MyStruct{}
+
+	err = PopulateStructFromJSON(data, &myStruct)
+	assert.NoError(t, err)
+
+	outData, err = SerializeStructToJSON(&myStruct)
+	assert.NoError(t, err)
+	assert.Equal(t, data, outData)
+
+	// Make sure that, without caching, unknown values are simply ignored
+	// without causing errors.
+	noCache := MyStructNoCache{}
+	err = PopulateStructFromJSON(data, &noCache)
+	assert.NoError(t, err)
+
+	expectedNoCache := []byte(`{"field0":"f1","field1":2}`)
+
+	outData, err = SerializeStructToJSON(&noCache)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedNoCache, outData)
+}