Db refactoring (#5)

* improve db access

* fix e2e

Co-authored-by: Victor Varza <[email protected]>

Author: victorvarza

cmd/api/api.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/adobe/cluster-registry/pkg/api/docs"
 	"github.com/adobe/cluster-registry/pkg/api/monitoring"
 	"github.com/adobe/cluster-registry/pkg/api/sqs"
+	"github.com/adobe/cluster-registry/pkg/api/utils"
 	echoSwagger "github.com/swaggo/echo-swagger"
 )
 
@@ -37,6 +38,8 @@ import (
 // @in header
 // @name Authorization
 func main() {
+
+	appConfig := utils.LoadApiConfig()
 	a := api.NewRouter()
 
 	a.GET("/api/swagger/*", echoSwagger.WrapHandler)
@@ -46,11 +49,11 @@ func main() {
 	m := monitoring.NewMetrics("cluster_registry_api", nil, false)
 	m.Use(a)
 
-	d := database.NewDb(m)
-	h := api.NewHandler(d, m)
+	d := database.NewDb(appConfig, m)
+	h := api.NewHandler(appConfig, d, m)
 	h.Register(v1)
 
-	c := sqs.NewConsumer(d, m)
+	c := sqs.NewConsumer(appConfig, d, m)
 	go c.Consume()
 
 	a.Logger.Fatal(a.Start(":8080"))

cmd/cc/client.go

@@ -17,6 +17,7 @@ import (
 	"os"
 
 	"github.com/adobe/cluster-registry/pkg/api/sqs"
+	"github.com/adobe/cluster-registry/pkg/api/utils"
 	"github.com/adobe/cluster-registry/pkg/cc/controllers"
 	"github.com/adobe/cluster-registry/pkg/cc/monitoring"
 
@@ -55,6 +56,7 @@ func main() {
 	var probeAddr string
 	var alertmanagerWebhookAddr string
 	var namespace string
+
 	flag.StringVar(&configFile, "config", "",
 		"The controller will load its initial configuration from this file. "+
 			"Omit this flag to use the default configuration values. "+
@@ -118,11 +120,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	appConfig := utils.LoadClientConfig()
+	sqsProducer := sqs.NewProducer(appConfig, m)
+
 	if err = (&controllers.ClusterReconciler{
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("controllers").WithName("Cluster"),
 		Scheme: mgr.GetScheme(),
-		Queue:  sqs.NewProducer(m),
+		Queue:  sqsProducer,
 		CAData: base64.StdEncoding.EncodeToString(mgr.GetConfig().CAData),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Cluster")

local/.env.local

@@ -4,6 +4,7 @@ export AWS_REGION="aws-region"
 export DB_AWS_REGION="db-aws-region"
 export DB_ENDPOINT="http://localhost:8000"
 export DB_TABLE_NAME="cluster-registry-local"
+export "DB_INDEX_NAME"="search-index-local"
 export SQS_ENDPOINT="http://localhost:9324"
 export SQS_AWS_REGION="sqs-aws-region"
 export SQS_QUEUE_NAME="cluster-registry-local"

local/db/import.go

@@ -8,14 +8,17 @@ import (
 
 	"github.com/adobe/cluster-registry/pkg/api/database"
 	"github.com/adobe/cluster-registry/pkg/api/monitoring"
+	"github.com/adobe/cluster-registry/pkg/api/utils"
 	registryv1 "github.com/adobe/cluster-registry/pkg/cc/api/registry/v1"
 	"gopkg.in/yaml.v2"
 )
 
 func main() {
 	var clusters []registryv1.Cluster
+
 	m := monitoring.NewMetrics("cluster_registry_api_local", nil, false)
-	d := database.NewDb(m)
+	appConfig := utils.LoadApiConfig()
+	d := database.NewDb(appConfig, m)
 
 	input_file := flag.String("input-file", "dummy-data.yaml", "yaml file path")
 	flag.Parse()

local/db/schema.json

@@ -1,19 +1,45 @@
 {
+    "TableName": "cluster-registry-local",
     "AttributeDefinitions": [
-        {
-            "AttributeName": "name",
-            "AttributeType": "S"
-        }
+      {
+        "AttributeName": "name",
+        "AttributeType": "S"
+      },
+      {
+        "AttributeName": "kind",
+        "AttributeType": "S"
+      }
     ],
     "ProvisionedThroughput": {
-        "WriteCapacityUnits": 5,
-        "ReadCapacityUnits": 5
+      "WriteCapacityUnits": 5,
+      "ReadCapacityUnits": 5
     },
-    "TableName": "cluster-registry-local",
     "KeySchema": [
-        {
-            "KeyType": "HASH",
-            "AttributeName": "name"
+      {
+        "KeyType": "HASH",
+        "AttributeName": "name"
+      }
+    ],
+    "GlobalSecondaryIndexes": [
+      {
+          "IndexName": "search-index-local",
+          "KeySchema": [
+            {
+              "AttributeName": "kind",
+              "KeyType": "HASH"
+            },
+            {
+              "AttributeName": "name",
+              "KeyType": "RANGE"
+            }
+          ],
+          "Projection": {
+            "ProjectionType": "ALL"
+          },
+          "ProvisionedThroughput": {
+            "ReadCapacityUnits": 5,
+            "WriteCapacityUnits": 5
+          }
         }
     ]
-}
+  }

local/setup.sh

@@ -81,11 +81,13 @@ if [[ "${RUN_API}" == 1 ]]; then
 	docker run -d \
 		--name ${CONTAINER_API} \
 		-p 8080:8080 \
+		-e AWS_REGION \
 		-e AWS_ACCESS_KEY_ID \
 		-e AWS_SECRET_ACCESS_KEY \
 		-e DB_AWS_REGION \
 		-e DB_ENDPOINT=http://${CONTAINER_DB}:8000 \
 		-e DB_TABLE_NAME=${DB_TABLE_NAME} \
+		-e DB_INDEX_NAME=${DB_INDEX_NAME} \
 		-e OIDC_ISSUER_URL=http://${CONTAINER_OIDC} \
 		-e OIDC_CLIENT_ID \
 		-e SQS_AWS_REGION \

pkg/api/api/handler.go

@@ -13,7 +13,6 @@ governing permissions and limitations under the License.
 package api
 
 import (
-	"fmt"
 	"net/http"
 	"strconv"
 
@@ -35,21 +34,23 @@ type Handler interface {
 
 // handler struct
 type handler struct {
-	db  database.Db
-	met monitoring.MetricsI
+	db        database.Db
+	appConfig *utils.AppConfig
+	metrics   monitoring.MetricsI
 }
 
 // NewHandler func
-func NewHandler(d database.Db, m monitoring.MetricsI) Handler {
+func NewHandler(appConfig *utils.AppConfig, d database.Db, m monitoring.MetricsI) Handler {
 	h := &handler{
-		db:  d,
-		met: m,
+		db:        d,
+		metrics:   m,
+		appConfig: appConfig,
 	}
 	return h
 }
 
 func (h *handler) Register(v1 *echo.Group) {
-	a, err := authz.NewAuthenticator(h.met)
+	a, err := authz.NewAuthenticator(h.appConfig, h.metrics)
 	if err != nil {
 		log.Fatalf("Failed to initialize authenticator: %v", err)
 	}
@@ -88,30 +89,27 @@ func (h *handler) GetCluster(ctx echo.Context) error {
 
 // ListClusters godoc
 // @Summary List all clusters
-// @Description List all clusters. Use query parameters to filter results. Auth is required
+// @Description List all clusters. Use query parametricsers to filter results. Auth is required
 // @ID get-clusters
 // @Tags cluster
 // @Accept  json
 // @Produce  json
 // @Param region query string false "Filter by region"
 // @Param environment query string false "Filter by environment"
-// @Param businessUnit query string false "Filter by businessUnit"
 // @Param status query string false "Filter by status"
-// @Param limit query integer false "Limit number of clusters returned (default is 10)"
-// @Param offset query integer false "Offset/skip number of clusters (default is 0)"
+// @Param offset query integer false "Offset to start pagination search results (default is 0)"
+// @Param limit query integer false "The number of results per page (default is 200)"
 // @Success 200 {object} clusterList
 // @Failure 500 {object} utils.Error
 // @Security bearerAuth
 // @Router /v1/clusters [get]
 func (h *handler) ListClusters(ctx echo.Context) error {
-	var (
-		clusters []registryv1.Cluster
-		count    int
-	)
 
-	region := ctx.QueryParam("region")
+	var clusters []registryv1.Cluster
+	var count int
+
 	environment := ctx.QueryParam("environment")
-	businessUnit := ctx.QueryParam("businessUnit")
+	region := ctx.QueryParam("region")
 	status := ctx.QueryParam("status")
 
 	offset, err := strconv.Atoi(ctx.QueryParam("offset"))
@@ -121,11 +119,9 @@ func (h *handler) ListClusters(ctx echo.Context) error {
 
 	limit, err := strconv.Atoi(ctx.QueryParam("limit"))
 	if err != nil {
-		limit = 20
+		limit = 200
 	}
 
-	fmt.Println(limit, offset)
-	clusters, count, _ = h.db.ListClusters(region, environment, businessUnit, status)
-
-	return ctx.JSON(http.StatusOK, newClusterListResponse(clusters, count))
+	clusters, count, more, _ := h.db.ListClusters(offset, limit, region, environment, status)
+	return ctx.JSON(http.StatusOK, newClusterListResponse(clusters, count, offset, limit, more))
 }

pkg/api/api/handler_test.go

@@ -20,13 +20,14 @@ import (
 
 	"github.com/adobe/cluster-registry/pkg/api/database"
 	"github.com/adobe/cluster-registry/pkg/api/monitoring"
+	"github.com/adobe/cluster-registry/pkg/api/utils"
 	registryv1 "github.com/adobe/cluster-registry/pkg/cc/api/registry/v1"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/labstack/echo/v4"
 	"github.com/stretchr/testify/assert"
 )
 
-// mockDatabase database.db
+// mockDatabase extends database.db
 type mockDatabase struct {
 	database.Db
 	clusters []registryv1.Cluster
@@ -41,20 +42,23 @@ func (m mockDatabase) GetCluster(name string) (*registryv1.Cluster, error) {
 	return nil, nil
 }
 
-func (m mockDatabase) ListClusters(region string, environment string, businessUnit string, status string) ([]registryv1.Cluster, int, error) {
-	return m.clusters, len(m.clusters), nil
+func (m mockDatabase) ListClusters(offset int, limit int, environment string, region string, status string) ([]registryv1.Cluster, int, bool, error) {
+	return m.clusters, len(m.clusters), false, nil
 }
 
 func TestNewHandler(t *testing.T) {
 	test := assert.New(t)
+	appConfig := &utils.AppConfig{}
 	d := mockDatabase{}
 	m := monitoring.NewMetrics("cluster_registry_api_handler_test", nil, true)
-	h := NewHandler(d, m)
+	h := NewHandler(appConfig, d, m)
 	test.NotNil(h)
 }
 
 func TestGetCluster(t *testing.T) {
 	test := assert.New(t)
+	appConfig := &utils.AppConfig{}
+
 	tcs := []struct {
 		name             string
 		clusterName      string
@@ -95,7 +99,7 @@ func TestGetCluster(t *testing.T) {
 
 		d := mockDatabase{clusters: tc.clusters}
 		m := monitoring.NewMetrics("cluster_registry_api_handler_test", nil, true)
-		h := NewHandler(d, m)
+		h := NewHandler(appConfig, d, m)
 		r := NewRouter()
 
 		req := httptest.NewRequest(echo.GET, "/api/v1/clusters/:name", nil)
@@ -123,6 +127,8 @@ func TestGetCluster(t *testing.T) {
 
 func TestListClusters(t *testing.T) {
 	test := assert.New(t)
+	appConfig := &utils.AppConfig{}
+
 	tcs := []struct {
 		name           string
 		clusters       []registryv1.Cluster
@@ -156,7 +162,7 @@ func TestListClusters(t *testing.T) {
 
 		d := mockDatabase{clusters: tc.clusters}
 		m := monitoring.NewMetrics("cluster_registry_api_handler_test", nil, true)
-		h := NewHandler(d, m)
+		h := NewHandler(appConfig, d, m)
 		r := NewRouter()
 
 		req := httptest.NewRequest(echo.GET, "/api/v1/clusters", nil)

pkg/api/api/response.go

@@ -19,23 +19,30 @@ import (
 
 type clusterList struct {
 	Items      []*registryv1.ClusterSpec `json:"items"`
-	ItemsCount int                       `json:"itemsCount"`
+	ItemsCount int                       `json:"itemsCount"` // TODO: should be rename to total
+	Offset     int                       `json:"offset"`
+	Limit      int                       `json:"limit"`
+	More       bool                      `json:"more"`
 }
 
 func newClusterResponse(ctx echo.Context, c *registryv1.Cluster) *registryv1.ClusterSpec {
 	cs := &c.Spec
 	return cs
 }
 
-func newClusterListResponse(clusters []registryv1.Cluster, count int) *clusterList {
+func newClusterListResponse(clusters []registryv1.Cluster, count int, offset int, limit int, more bool) *clusterList {
 	r := new(clusterList)
-	r.Items = make([]*registryv1.ClusterSpec, 0)
+	r.Items = make([]*registryv1.ClusterSpec, 0) // TODO: check memory allocation
 
 	for _, c := range clusters {
 		cs := c.Spec
 		r.Items = append(r.Items, &cs)
 	}
 
 	r.ItemsCount = count
+	r.Offset = offset
+	r.Limit = limit
+	r.More = more
+
 	return r
 }

pkg/api/authz/jwt.go

@@ -16,7 +16,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"os"
 	"time"
 
 	"github.com/adobe/cluster-registry/pkg/api/monitoring"
@@ -33,8 +32,6 @@ const (
 )
 
 var (
-	clientID    = os.Getenv("OIDC_CLIENT_ID")
-	issuerURL   = os.Getenv("OIDC_ISSUER_URL")
 	tokenLookup = "Authorization"
 	authScheme  = "Bearer"
 )
@@ -47,16 +44,16 @@ type Authenticator struct {
 }
 
 // NewAuthenticator creates new Authenticator
-func NewAuthenticator(m monitoring.MetricsI) (*Authenticator, error) {
+func NewAuthenticator(appConfig *utils.AppConfig, m monitoring.MetricsI) (*Authenticator, error) {
 	ctx := context.Background()
-	provider, err := oidc.NewProvider(ctx, issuerURL)
+	provider, err := oidc.NewProvider(ctx, appConfig.OidcIssuerUrl)
 
 	if err != nil {
 		return nil, fmt.Errorf("init verifier failed: %v", err)
 	}
 
 	config := &oidc.Config{
-		ClientID: clientID,
+		ClientID: appConfig.OidcClientId,
 	}
 
 	verifier := provider.Verifier(config)