Merge pull request #89 from kaleido-io/always-paginate

config for always returning pagination response

Author: peterbroadhurst

pkg/ffapi/apirequest.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Kaleido, Inc.
+// Copyright © 2023 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -31,17 +31,24 @@ type APIRequest struct {
 	Part            *Multipart
 	SuccessStatus   int
 	ResponseHeaders http.Header
+	AlwaysPaginate  bool
 }
 
 // FilterResult is a helper to transform a filter result into a REST API standard payload
 func (r *APIRequest) FilterResult(items interface{}, res *FilterResult, err error) (interface{}, error) {
 	itemsVal := reflect.ValueOf(items)
-	if err != nil || res == nil || res.TotalCount == nil || itemsVal.Kind() != reflect.Slice {
-		return items, err
+	if itemsVal.Kind() == reflect.Slice && (r.AlwaysPaginate || (res != nil && res.TotalCount != nil)) {
+		response := &FilterResultsWithCount{
+			Items: items,
+		}
+		if res != nil {
+			response.Total = res.TotalCount
+		}
+		if itemsVal.Kind() == reflect.Slice {
+			response.Count = int64(itemsVal.Len())
+		}
+		return response, err
 	}
-	return &FilterResultsWithCount{
-		Total: *res.TotalCount,
-		Count: int64(itemsVal.Len()),
-		Items: items,
-	}, nil
+	return items, err
+
 }

pkg/ffapi/apirequest_test.go

@@ -31,7 +31,7 @@ func TestFilterResultWithCount(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, &FilterResultsWithCount{
 		Count: 1,
-		Total: 10,
+		Total: &ten,
 		Items: []string{"test"},
 	}, f)
 }
@@ -42,3 +42,36 @@ func TestFilterResultPlain(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []string{"test"}, f)
 }
+
+func TestFilterResultWithCountNoTotal(t *testing.T) {
+	r := &APIRequest{
+		AlwaysPaginate: true,
+	}
+	f, err := r.FilterResult([]string{"test"}, &FilterResult{}, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, &FilterResultsWithCount{
+		Count: 1,
+		Items: []string{"test"},
+	}, f)
+}
+
+func TestFilterResultAlwyasPaginateNoSlice(t *testing.T) {
+	r := &APIRequest{
+		AlwaysPaginate: true,
+	}
+	f, err := r.FilterResult("test", &FilterResult{}, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", f)
+}
+
+func TestFilterResultAlwaysPaginateNoFilterResult(t *testing.T) {
+	r := &APIRequest{
+		AlwaysPaginate: true,
+	}
+	f, err := r.FilterResult([]string{"test"}, nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, &FilterResultsWithCount{
+		Count: 1,
+		Items: []string{"test"},
+	}, f)
+}

pkg/ffapi/apiserver.go

@@ -59,6 +59,7 @@ type apiServer[T any] struct {
 	requestTimeout     time.Duration
 	requestMaxTimeout  time.Duration
 	apiPublicURL       string
+	alwaysPaginate     bool
 	metricsEnabled     bool
 	metricsPath        string
 	metricsPublicURL   string
@@ -89,12 +90,16 @@ type APIServerRouteExt[T any] struct {
 // the supplied wrapper function - which will inject
 func NewAPIServer[T any](ctx context.Context, options APIServerOptions[T]) APIServer {
 	as := &apiServer[T]{
-		requestTimeout:    options.APIConfig.GetDuration(ConfAPIRequestTimeout),
-		requestMaxTimeout: options.APIConfig.GetDuration(ConfAPIRequestMaxTimeout),
-		metricsEnabled:    options.MetricsConfig.GetBool(ConfMetricsServerEnabled),
-		metricsPath:       options.MetricsConfig.GetString(ConfMetricsServerPath),
-		APIServerOptions:  options,
-		started:           make(chan struct{}),
+		defaultFilterLimit: options.APIConfig.GetUint64(ConfAPIDefaultFilterLimit),
+		maxFilterLimit:     options.APIConfig.GetUint64(ConfAPIMaxFilterLimit),
+		maxFilterSkip:      options.APIConfig.GetUint64(ConfAPIMaxFilterSkip),
+		requestTimeout:     options.APIConfig.GetDuration(ConfAPIRequestTimeout),
+		requestMaxTimeout:  options.APIConfig.GetDuration(ConfAPIRequestMaxTimeout),
+		metricsEnabled:     options.MetricsConfig.GetBool(ConfMetricsServerEnabled),
+		metricsPath:        options.MetricsConfig.GetString(ConfMetricsServerPath),
+		alwaysPaginate:     options.APIConfig.GetBool(ConfAPIAlwaysPaginate),
+		APIServerOptions:   options,
+		started:            make(chan struct{}),
 	}
 	if as.FavIcon16 == nil {
 		as.FavIcon16 = ffLogo16
@@ -239,6 +244,7 @@ func (as *apiServer[T]) handlerFactory() *HandlerFactory {
 		MaxFilterSkip:         as.maxFilterSkip,
 		MaxFilterLimit:        as.maxFilterLimit,
 		SupportFieldRedaction: as.SupportFieldRedaction,
+		AlwaysPaginate:        as.alwaysPaginate,
 	}
 }
 

pkg/ffapi/apiserver_config.go

@@ -30,6 +30,7 @@ var (
 	ConfAPIMaxFilterSkip      = "maxFilterSkip"
 	ConfAPIRequestTimeout     = "requestTimeout"
 	ConfAPIRequestMaxTimeout  = "requestMaxTimeout"
+	ConfAPIAlwaysPaginate     = "alwaysPaginate"
 )
 
 func InitAPIServerConfig(apiConfig, metricsConfig, corsConfig config.Section) {
@@ -39,6 +40,7 @@ func InitAPIServerConfig(apiConfig, metricsConfig, corsConfig config.Section) {
 	apiConfig.AddKnownKey(ConfAPIMaxFilterSkip, 100000)
 	apiConfig.AddKnownKey(ConfAPIRequestTimeout, "30s")
 	apiConfig.AddKnownKey(ConfAPIRequestMaxTimeout, "10m")
+	apiConfig.AddKnownKey(ConfAPIAlwaysPaginate, false)
 
 	httpserver.InitCORSConfig(corsConfig)
 

pkg/ffapi/handler.go

@@ -48,6 +48,7 @@ type HandlerFactory struct {
 	MaxFilterSkip         uint64
 	MaxFilterLimit        uint64
 	PassthroughHeaders    []string
+	AlwaysPaginate        bool
 	SupportFieldRedaction bool
 }
 
@@ -172,6 +173,7 @@ func (hs *HandlerFactory) RouteHandler(route *Route) http.HandlerFunc {
 				Input:           jsonInput,
 				SuccessStatus:   http.StatusOK,
 				ResponseHeaders: res.Header(),
+				AlwaysPaginate:  hs.AlwaysPaginate,
 			}
 			if len(route.JSONOutputCodes) > 0 {
 				r.SuccessStatus = route.JSONOutputCodes[0]

pkg/ffapi/restfilter.go

@@ -31,7 +31,7 @@ import (
 
 type FilterResultsWithCount struct {
 	Count int64       `json:"count"`
-	Total int64       `json:"total"`
+	Total *int64      `json:"total,omitempty"` // omitted if a count was not calculated (AlwaysPaginate enabled, and count not specified)
 	Items interface{} `json:"items"`
 }