Image Super Cache

Performance observations from benchmarking image reads showed that we
were spending a very large amount of time converting images, part of the
solution to this is to actually do the conversion on read and cache that
result.  There is also a fairly significant heap/garbage collection
penalty too.

The other important observation is that on a cache refresh someone has
to incur the cost.  If we blocked start up to allow caches to be
pre-heated, and also performed the refreshes in the background, then no
one has to suffer.

This change replaces the toy cache with a fully fledged refresh aside
implementation.  Performance metrics show a 66% time reduction in
reading images, mostly due presumably to the conversion being cached,
but also aided by new zero-copy semantics.

The refresh aside cache also supports data epochs, so we pass the whole
lot up to the APIs which can now combine that with any query related
data and cache the JSON formatted data, which would save another huge
amount of cycles and memory.

Author: spjmurray

go.mod

@@ -15,8 +15,8 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/spjmurray/go-util v0.1.3
 	github.com/stretchr/testify v1.11.1
-	github.com/unikorn-cloud/core v1.14.0-rc1
-	github.com/unikorn-cloud/identity v1.14.0-rc1
+	github.com/unikorn-cloud/core v1.14.0-rc1.0.20260213113949-5c30aaff8409
+	github.com/unikorn-cloud/identity v1.14.0-rc1.0.20260209130807-b25e8c7ed9ac
 	go.opentelemetry.io/otel v1.35.0
 	go.opentelemetry.io/otel/trace v1.35.0
 	go.uber.org/mock v0.5.2

go.sum

@@ -203,10 +203,10 @@ github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE
 github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/unikorn-cloud/core v1.14.0-rc1 h1:5kq5v3hjXnDNPBy1BsDLSwY233u8XLYwyleywJeTFUo=
-github.com/unikorn-cloud/core v1.14.0-rc1/go.mod h1:NBcUrdV7RKIPFBdootNeN74otIoNLQp14+KpVE5m1X0=
-github.com/unikorn-cloud/identity v1.14.0-rc1 h1:B/G9Lp27qPQHnKPNrfOJlY7YnahMSlv38cwtHC/mzkY=
-github.com/unikorn-cloud/identity v1.14.0-rc1/go.mod h1:6mJqFyX8QZHvNFV0aYZJrE/1eozR5xiZYORNxe1Xlxc=
+github.com/unikorn-cloud/core v1.14.0-rc1.0.20260213113949-5c30aaff8409 h1:7yYJHVFjebQF+Nf5K3bR7QyGlVKlxqTXTfDauoGoIsI=
+github.com/unikorn-cloud/core v1.14.0-rc1.0.20260213113949-5c30aaff8409/go.mod h1:NBcUrdV7RKIPFBdootNeN74otIoNLQp14+KpVE5m1X0=
+github.com/unikorn-cloud/identity v1.14.0-rc1.0.20260209130807-b25e8c7ed9ac h1:jsby+/f0ZP6Dr7riZqjuQ3tAAuw9g6W++N3n4ERyX34=
+github.com/unikorn-cloud/identity v1.14.0-rc1.0.20260209130807-b25e8c7ed9ac/go.mod h1:ogNk6VL1iWYd7UOfn9jgm/Cd8Q8aXzaOhrsg+5jGrtc=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/handler/handler_v2_image_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 
+	"github.com/unikorn-cloud/core/pkg/util/cache"
 	"github.com/unikorn-cloud/region/pkg/handler/common"
 	imagemock "github.com/unikorn-cloud/region/pkg/handler/image/mock"
 	"github.com/unikorn-cloud/region/pkg/openapi"
@@ -57,7 +58,7 @@ func Test_Imagev2_List(t *testing.T) {
 		"available to org": {
 			setupQuery: func(query *imagemock.MockImageQuery) {
 				query.EXPECT().AvailableToOrganization(orgID).Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 			params: openapi.GetApiV2RegionsRegionIDImagesParams{
 				OrganizationID: ptr.To([]string{orgID}),
@@ -67,7 +68,7 @@ func Test_Imagev2_List(t *testing.T) {
 			setupQuery: func(query *imagemock.MockImageQuery) {
 				query.EXPECT().AvailableToOrganization(orgID).Return(query)
 				query.EXPECT().StatusIn(types.ImageStatusReady).Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 			params: openapi.GetApiV2RegionsRegionIDImagesParams{
 				OrganizationID: ptr.To([]string{orgID}),
@@ -77,7 +78,7 @@ func Test_Imagev2_List(t *testing.T) {
 		"owned by org": {
 			setupQuery: func(query *imagemock.MockImageQuery) {
 				query.EXPECT().OwnedByOrganization(orgID).Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 			params: openapi.GetApiV2RegionsRegionIDImagesParams{
 				OrganizationID: ptr.To([]string{orgID}),
@@ -88,7 +89,7 @@ func Test_Imagev2_List(t *testing.T) {
 			setupQuery: func(query *imagemock.MockImageQuery) {
 				query.EXPECT().AvailableToOrganization(orgID).Return(query)
 				query.EXPECT().StatusIn(types.ImageStatusReady).Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 			params: openapi.GetApiV2RegionsRegionIDImagesParams{
 				OrganizationID: ptr.To([]string{orgID}),
@@ -101,7 +102,7 @@ func Test_Imagev2_List(t *testing.T) {
 				// This must be called even though the caller has no permission to the org,
 				// to get global images, since those are counted as available.
 				query.EXPECT().AvailableToOrganization().Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 			params: openapi.GetApiV2RegionsRegionIDImagesParams{
 				OrganizationID: ptr.To([]string{"NOT" + orgID}),
@@ -110,7 +111,7 @@ func Test_Imagev2_List(t *testing.T) {
 		"no filters": { // when asking without giving an organization, you don't need org permissions.
 			setupQuery: func(query *imagemock.MockImageQuery) {
 				query.EXPECT().AvailableToOrganization().Return(query)
-				query.EXPECT().List(gomock.Any()).Return(nil, nil)
+				query.EXPECT().List().Return(&cache.ListSnapshot[types.Image]{}, nil)
 			},
 		},
 	}

pkg/handler/image/conversions.go

@@ -185,11 +185,11 @@ func convertImage(in *types.Image) *openapi.Image {
 	return out
 }
 
-func convertImages(in []types.Image) openapi.Images {
-	out := make(openapi.Images, len(in))
+func convertImages(in types.ImageList) openapi.Images {
+	out := make(openapi.Images, len(in.Items))
 
-	for i := range in {
-		out[i] = *convertImage(&in[i])
+	for i := range in.Items {
+		out[i] = *convertImage(in.Items[i])
 	}
 
 	return out

pkg/handler/image/image.go

@@ -64,14 +64,14 @@ func (c *Client) ListImages(ctx context.Context, organizationID, regionID string
 	result, err := query.
 		AvailableToOrganization(organizationID).
 		StatusIn(types.ImageStatusReady).
-		List(ctx)
+		List()
 
 	if err != nil {
 		return nil, err
 	}
 
 	// Apply ordering guarantees, ordered by name.
-	slices.SortStableFunc(result, func(a, b types.Image) int {
+	slices.SortStableFunc(result.Items, func(a, b *types.Image) int {
 		return cmp.Compare(a.Name, b.Name)
 	})
 

pkg/handler/image/mock/queryinterfaces.go

@@ -56,13 +56,13 @@ func (c *Client) QueryImages(ctx context.Context, regionID string, params openap
 		query = query.StatusIn(queryStatuses...)
 	}
 
-	result, err := query.List(ctx)
+	result, err := query.List()
 	if err != nil {
 		return nil, err
 	}
 
 	// Apply ordering guarantees, ordered by name.
-	slices.SortStableFunc(result, func(a, b types.Image) int {
+	slices.SortStableFunc(result.Items, func(a, b *types.Image) int {
 		return cmp.Compare(a.Name, b.Name)
 	})
 

pkg/handler/image/query.go

@@ -21,12 +21,12 @@ import (
 	"context"
 
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
-	"github.com/gophercloud/gophercloud/v2/openstack/image/v2/images"
 	"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/routers"
 	"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/groups"
 	"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports"
 	"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets"
 
+	"github.com/unikorn-cloud/core/pkg/util/cache"
 	unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1"
 	"github.com/unikorn-cloud/region/pkg/providers/types"
 
@@ -35,7 +35,7 @@ import (
 
 // NewImageQuery makes the internal implementation of ImageQuery available for testing
 // on its own.
-func NewImageQuery(listFunc func(context.Context) ([]images.Image, error)) types.ImageQuery {
+func NewImageQuery(listFunc func() (*cache.ListSnapshot[types.Image], error)) types.ImageQuery {
 	return &imageQuery{listFunc: listFunc}
 }