feat: implement extension caching

Extensions objects will now cache any extensions they don't recognize
(that don't correspond to a field inside their registered IMapValue)
when deserializing values.

This means CORIMs with extensions remain stable when deserialized and
then re-serialized using structs without registered extensions.

Cached extensions are only used during marshalling, and will not be
returned when calling the Get* methods. However they can be accessed
directly via Extensions.Cached field.

When an IMapValue struct is registered, cached values are scanned, and
the new struct is populated with cached values, which are then removed
form the cache.

When registering a new struct when there is an existing IMapValue, the
non-zero-value fields of the old IMapValue are stored in the cache.

Signed-off-by: Sergei Trofimov <[email protected]>

Author: setrofim

extensions/README.md

@@ -127,6 +127,20 @@ method, passing itself as the parameter.
 You do not need to define this method unless you actually want to enforce some
 constraints (i.e., if you just want to define additional fields).
 
+### Unknown extensions caching
+
+When unmarshaled data contains entries that do not correspond to fields inside
+a registered extensions struct, their values get cached inside the `Extensions`
+objects. If the containing object is later re-marshalled, cached values will be
+included, so that unknown extensions are not lost.
+
+Cached extension values are not accessible via `Get*()` methods described
+above, however they are available as `Extensions.Cached` map.
+
+If an extensions struct is registered after unmarshalling, it will be populated
+with any now-recognized cached values, which will then be removed from the
+cache.
+
 ### Example
 
 The following example illustrates how to implement a map extension by extending

extensions/extensions.go

@@ -1,4 +1,4 @@
-// Copyright 2023-2024 Contributors to the Veraison project.
+// Copyright 2023-2025 Contributors to the Veraison project.
 // SPDX-License-Identifier: Apache-2.0
 package extensions
 
@@ -24,14 +24,24 @@ type ExtensionValue struct {
 
 type Extensions struct {
 	IMapValue `json:"extensions,omitempty"`
+
+	Cached map[string]any `field-cache:"" cbor:"-" json:"-"`
 }
 
 func (o *Extensions) Register(exts IMapValue) {
 	if reflect.TypeOf(exts).Kind() != reflect.Pointer {
 		panic("attempting to register a non-pointer IMapValue")
 	}
 
+	// Ensure that the values of any existing extensions are preserved.
+	// The contents of the existing IMapValue (if there is one) are added
+	// to the cache, which is then applied to the new IMapValue. If the new
+	// IMapValue has fields corresponding to the old extensions, they will be
+	// populated into the new IMapValue; any old extensions that are not
+	// recognized by the new IMapValue will be cached.
+	updateMapFromInterface(&o.Cached, o.IMapValue)
 	o.IMapValue = exts
+	updateInterfaceFromMap(o.IMapValue, o.Cached)
 }
 
 func (o *Extensions) HaveExtensions() bool {
@@ -455,3 +465,89 @@ func newIMapValue(v IMapValue) IMapValue {
 
 	return reflect.New(valType).Interface()
 }
+
+func updateMapFromInterface(mp *map[string]any, iface any) { // nolint: gocritic
+	if iface == nil {
+		return
+	}
+
+	if *mp == nil {
+		*mp = make(map[string]any)
+	}
+
+	ifType := reflect.TypeOf(iface)
+	ifVal := reflect.ValueOf(iface)
+	if ifType.Kind() == reflect.Pointer {
+		ifType = ifType.Elem()
+		ifVal = ifVal.Elem()
+	}
+
+	for i := 0; i < ifVal.NumField(); i++ {
+		typeField := ifType.Field(i)
+		tag, ok := typeField.Tag.Lookup("cbor")
+		if !ok {
+			continue
+		}
+
+		codePointText := strings.Split(tag, ",")[0]
+		valField := ifVal.Field(i)
+		if !valField.IsZero() {
+			(*mp)[codePointText] = valField.Interface()
+		}
+	}
+}
+
+func updateInterfaceFromMap(iface any, m map[string]any) {
+	if iface == nil {
+		panic("nil interface")
+	}
+
+	ifType := reflect.TypeOf(iface)
+	if ifType.Kind() != reflect.Pointer {
+		panic("interface must be a pointer")
+	}
+
+	ifType = ifType.Elem()
+	ifVal := reflect.ValueOf(iface).Elem()
+
+	for i := 0; i < ifVal.NumField(); i++ {
+		var fieldJSONTag, fieldCBORTag string
+		typeField := ifType.Field(i)
+		valField := ifVal.Field(i)
+
+		tag, ok := typeField.Tag.Lookup("json")
+		if ok {
+			fieldJSONTag = strings.Split(tag, ",")[0]
+		}
+
+		tag, ok = typeField.Tag.Lookup("cbor")
+		if ok {
+			fieldCBORTag = strings.Split(tag, ",")[0]
+		}
+
+		mapKey := fieldJSONTag
+		rawMapVal, ok := m[mapKey]
+		if !ok {
+			mapKey = fieldCBORTag
+			rawMapVal, ok = m[mapKey]
+			if !ok {
+				continue
+			}
+		}
+
+		mapVal := reflect.ValueOf(rawMapVal)
+		if !mapVal.Type().AssignableTo(typeField.Type) {
+			if mapVal.Type().ConvertibleTo(typeField.Type) {
+				mapVal = mapVal.Convert(typeField.Type)
+			} else {
+				// We cannot return an error here, and we don't
+				// want to panic, so we're just going to keep the
+				// entry in the cache.
+				continue
+			}
+		}
+
+		valField.Set(mapVal)
+		delete(m, mapKey)
+	}
+}

extensions/extensions_test.go

@@ -1,4 +1,4 @@
-// Copyright 2023-2024 Contributors to the Veraison project.
+// Copyright 2023-2025 Contributors to the Veraison project.
 // SPDX-License-Identifier: Apache-2.0
 package extensions
 
@@ -7,11 +7,12 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/veraison/corim/encoding"
 )
 
 type Entity struct {
-	EntityName string
-	Roles      []int64
+	EntityName string  `cbor:"0,keyasint" json:"entity-name"`
+	Roles      []int64 `cbor:"1,keyasint,omitempty" json:"roles,omitempty"`
 
 	Extensions
 }
@@ -168,3 +169,114 @@ func Test_Extensions_Values(t *testing.T) {
 	vals = exts.Values()
 	assert.Len(t, vals, 0)
 }
+
+func Test_Extensions_unknown_handling_CBOR(t *testing.T) {
+	// nolint: gocritic
+	data := []byte{
+		0xa4, // map(4) [entity]
+
+		0x00,             // key: 0 [entity-name]
+		0x63,             // value: tstr(3)
+		0x66, 0x6f, 0x6f, // "foo"
+
+		0x1,        // key: 1 [roles]
+		0x82,       // value: array(2)
+		0x01, 0x02, // [1, 2]
+
+		0x21, // key: -2 [extension(size)]
+		0x07, // value: 7
+
+		0x27, // key: -8 [extension(<unknown>)]
+		0xf5, // value: true
+	}
+
+	entity := Entity{}
+	err := encoding.PopulateStructFromCBOR(dm, data, &entity)
+	assert.NoError(t, err)
+	assert.Equal(t, "foo", entity.EntityName)
+	assert.Equal(t, []int64{1, 2}, entity.Roles)
+	assert.Equal(t, uint64(7), entity.Extensions.Cached["-2"]) // nolint: staticcheck
+
+	// Check that the cached value has been populated into the
+	// newly-registered struct.
+	entity.Register(&TestExtensions{})
+	assert.Equal(t, 7, entity.MustGetInt("size"))
+
+	// Check that the populated value is no longer cached.
+	_, ok := entity.Extensions.Cached["-2"] // nolint: staticcheck
+	assert.False(t, ok)
+
+	entity = Entity{}
+	entity.Register(&TestExtensions{})
+	err = encoding.PopulateStructFromCBOR(dm, data, &entity)
+	assert.NoError(t, err)
+
+	// If extensions were registered before unmarshalling, the value gets
+	// populated directly into the registered struct, bypassing the cache.
+	assert.Equal(t, 7, entity.MustGetInt("size"))
+	_, ok = entity.Extensions.Cached["-2"] // nolint: staticcheck
+	assert.False(t, ok)
+
+	// Values for keys in in the registered struct still go into cache.
+	val, ok := entity.Extensions.Cached["-8"] // nolint: staticcheck
+	assert.True(t, ok)
+	assert.True(t, val.(bool))
+
+	encoded, err := encoding.SerializeStructToCBOR(em, &entity)
+	assert.NoError(t, err)
+	assert.Equal(t, data, encoded)
+}
+
+func Test_Extensions_unknown_handling_JSON(t *testing.T) {
+	data := []byte(`{"entity-name":"foo","roles":[1,2],"size":7,"-8":true}`)
+
+	entity := Entity{}
+	err := encoding.PopulateStructFromJSON(data, &entity)
+	assert.NoError(t, err)
+	assert.Equal(t, "foo", entity.EntityName)
+	assert.Equal(t, []int64{1, 2}, entity.Roles)
+	assert.Equal(t, float64(7), entity.Extensions.Cached["size"]) // nolint: staticcheck
+
+	// since we only have the JSON field name "size", and we don't know
+	// what extension it corresponds to, CBOR encoding fails.
+	_, err = encoding.SerializeStructToCBOR(em, &entity)
+	assert.ErrorContains(t, err, "cached field name not an integer")
+
+	// Check that the cached value has been populated into the
+	// newly-registered struct.
+	entity.Register(&TestExtensions{})
+	assert.Equal(t, 7, entity.MustGetInt("size"))
+
+	// Check that the populated value is no longer cached.
+	_, ok := entity.Extensions.Cached["size"] // nolint: staticcheck
+	assert.False(t, ok)
+
+	// "size" has been recognized and removed form cache; we can now
+	// serialize it to CBOR as we now know its code point. The only
+	// remaining unknown extension has a name that can parse to an integer,
+	// so we can use that as the code point for CBOR, and serialization
+	// should succeed.
+	_, err = encoding.SerializeStructToCBOR(em, &entity)
+	assert.NoError(t, err)
+
+	entity = Entity{}
+	entity.Register(&TestExtensions{})
+	err = encoding.PopulateStructFromJSON(data, &entity)
+	assert.NoError(t, err)
+
+	// If extensions were registered before unmarshalling, the value gets
+	// populated directly into the registered struct, bypassing the cache.
+	assert.Equal(t, 7, entity.MustGetInt("size"))
+	_, ok = entity.Extensions.Cached["size"] // nolint: staticcheck
+	assert.False(t, ok)
+
+	// Values for keys in in the registered struct still go into cache.
+	val, ok := entity.Extensions.Cached["-8"] // nolint: staticcheck
+	assert.True(t, ok)
+	assert.True(t, val.(bool))
+
+	encoded, err := encoding.SerializeStructToJSON(&entity)
+	assert.NoError(t, err)
+
+	assert.JSONEq(t, string(data), string(encoded))
+}