Merge pull request #62 from hyperledger/dbsqltest

Expose the mock provider for other packages to use in tests

Author: peterbroadhurst

.vscode/settings.json

@@ -25,6 +25,7 @@
     "Infof",
     "Kaleido",
     "microservices",
+    "Nillable",
     "Nowarn",
     "openapi",
     "passwordfile",

pkg/dbsql/crud.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"reflect"
 	"strings"
 
 	sq "github.com/Masterminds/squirrel"
@@ -29,6 +30,12 @@ import (
 	"github.com/hyperledger/firefly-common/pkg/log"
 )
 
+const (
+	ColumnID      = "id"
+	ColumnCreated = "created"
+	ColumnUpdated = "updated"
+)
+
 type ChangeEventType int
 
 const (
@@ -45,17 +52,58 @@ const (
 	UpsertOptimizationExisting
 )
 
+type GetOption int
+
+const (
+	FailIfNotFound GetOption = iota
+)
+
 type PostCompletionHook func()
 
-type WithID interface {
+type Resource interface {
 	GetID() *fftypes.UUID
+	SetCreated(*fftypes.FFTime)
+	SetUpdated(*fftypes.FFTime)
 }
 
-type CrudBase[T WithID] struct {
-	DB             *Database
-	Table          string
-	Columns        []string
-	FilterFieldMap map[string]string
+type ResourceBase struct {
+	ID      *fftypes.UUID   `ffstruct:"ResourceBase" json:"id"`
+	Created *fftypes.FFTime `ffstruct:"ResourceBase" json:"created"`
+	Updated *fftypes.FFTime `ffstruct:"ResourceBase" json:"updated"`
+}
+
+func (r *ResourceBase) GetID() *fftypes.UUID {
+	return r.ID
+}
+
+func (r *ResourceBase) SetCreated(t *fftypes.FFTime) {
+	r.Created = t
+}
+
+func (r *ResourceBase) SetUpdated(t *fftypes.FFTime) {
+	r.Updated = t
+}
+
+type CRUD[T Resource] interface {
+	Validate()
+	Upsert(ctx context.Context, inst T, optimization UpsertOptimization, hooks ...PostCompletionHook) (created bool, err error)
+	InsertMany(ctx context.Context, instances []T, allowPartialSuccess bool, hooks ...PostCompletionHook) (err error)
+	Insert(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
+	Replace(ctx context.Context, inst T, hooks ...PostCompletionHook) (err error)
+	GetByID(ctx context.Context, id *fftypes.UUID, getOpts ...GetOption) (inst T, err error)
+	GetMany(ctx context.Context, filter ffapi.Filter) (instances []T, fr *ffapi.FilterResult, err error)
+	Update(ctx context.Context, id *fftypes.UUID, update ffapi.Update, hooks ...PostCompletionHook) (err error)
+	UpdateSparse(ctx context.Context, sparseUpdate T, hooks ...PostCompletionHook) (err error)
+	UpdateMany(ctx context.Context, filter ffapi.Filter, update ffapi.Update, hooks ...PostCompletionHook) (err error)
+	Delete(ctx context.Context, id *fftypes.UUID, hooks ...PostCompletionHook) (err error)
+}
+
+type CrudBase[T Resource] struct {
+	DB               *Database
+	Table            string
+	Columns          []string
+	FilterFieldMap   map[string]string
+	ImmutableColumns []string
 
 	NilValue     func() T // nil value typed to T
 	NewInstance  func() T
@@ -69,6 +117,57 @@ type CrudBase[T WithID] struct {
 	ReadQueryModifier func(sq.SelectBuilder) sq.SelectBuilder
 }
 
+// Validate checks things that must be true about a CRUD collection using this framework.
+// Intended for use in the unit tests of microservices (will exercise all the functions of the CrudBase):
+// - the mandatory columns exist - id/created/updated
+// - no column has the same name as the sequence column for the DB
+// - a unique pointer is returned for each field column
+// - the immutable columns exist
+// - the other functions return valid data
+func (c *CrudBase[T]) Validate() {
+	inst := c.NewInstance()
+	if isNil(inst) {
+		panic("NewInstance() value must not be nil")
+	}
+	ptrs := map[string]interface{}{}
+	fieldMap := map[string]bool{
+		// Mandatory column checks
+		ColumnID:      false,
+		ColumnCreated: false,
+		ColumnUpdated: false,
+	}
+	for _, col := range c.Columns {
+		if ok, set := fieldMap[col]; ok && set {
+			panic(fmt.Sprintf("%s is a duplicated column", col))
+		}
+
+		fieldMap[col] = true
+		if col == c.DB.sequenceColumn {
+			panic(fmt.Sprintf("cannot have column named '%s'", c.DB.sequenceColumn))
+		}
+		fieldPtr := c.GetFieldPtr(inst, col)
+		ptrVal := reflect.ValueOf(fieldPtr)
+		if ptrVal.Kind() != reflect.Ptr || !isNil(ptrVal.Elem().Interface()) {
+			panic(fmt.Sprintf("field %s does not seem to be a pointer type - prevents null-check for PATCH semantics functioning", col))
+		}
+		ptrs[col] = fieldPtr
+	}
+	for col, set := range fieldMap {
+		if !set {
+			panic(fmt.Sprintf("mandatory column '%s' must be included in column list", col))
+		}
+	}
+	if !isNil(c.NilValue()) {
+		panic("NilValue() value must be nil")
+	}
+	if isNil(c.ScopedFilter()) {
+		panic("ScopedFilter() value must not be nil")
+	}
+	if !isNil(c.GetFieldPtr(inst, fftypes.NewUUID().String())) {
+		panic("GetFieldPtr() must return nil for unknown column")
+	}
+}
+
 func (c *CrudBase[T]) idFilter(id *fftypes.UUID) sq.Eq {
 	filter := c.ScopedFilter()
 	if c.ReadTableAlias != "" {
@@ -79,11 +178,27 @@ func (c *CrudBase[T]) idFilter(id *fftypes.UUID) sq.Eq {
 	return filter
 }
 
-func (c *CrudBase[T]) attemptReplace(ctx context.Context, tx *TXWrapper, inst T) (int64, error) {
-	update := sq.Update(c.Table)
+func (c *CrudBase[T]) buildUpdateList(_ context.Context, update sq.UpdateBuilder, inst T, includeNil bool) sq.UpdateBuilder {
+colLoop:
 	for _, col := range c.Columns {
-		update = update.Set(col, c.GetFieldPtr(inst, col))
+		for _, immutable := range append(c.ImmutableColumns, ColumnID, ColumnCreated, ColumnUpdated, c.DB.sequenceColumn) {
+			if col == immutable {
+				continue colLoop
+			}
+		}
+		value := c.getFieldValue(inst, col)
+		if includeNil || !isNil(value) {
+			update = update.Set(col, value)
+		}
 	}
+	update = update.Set(ColumnUpdated, fftypes.Now())
+	return update
+}
+
+func (c *CrudBase[T]) updateFromInstance(ctx context.Context, tx *TXWrapper, inst T, includeNil bool) (int64, error) {
+	update := sq.Update(c.Table)
+	inst.SetUpdated(fftypes.Now())
+	update = c.buildUpdateList(ctx, update, inst, includeNil)
 	update = update.Where(c.idFilter(inst.GetID()))
 	return c.DB.UpdateTx(ctx, c.Table, tx,
 		update,
@@ -94,11 +209,19 @@ func (c *CrudBase[T]) attemptReplace(ctx context.Context, tx *TXWrapper, inst T)
 		})
 }
 
+func (c *CrudBase[T]) getFieldValue(inst T, col string) interface{} {
+	// Validate() will have checked this is safe for microservices (as long as they use that at build time in their UTs)
+	return reflect.ValueOf(c.GetFieldPtr(inst, col)).Elem().Interface()
+}
+
 func (c *CrudBase[T]) attemptInsert(ctx context.Context, tx *TXWrapper, inst T, requestConflictEmptyResult bool) (err error) {
+	now := fftypes.Now()
+	inst.SetCreated(now)
+	inst.SetUpdated(now)
 	insert := sq.Insert(c.Table).Columns(c.Columns...)
 	values := make([]interface{}, len(c.Columns))
 	for i, col := range c.Columns {
-		values[i] = c.GetFieldPtr(inst, col)
+		values[i] = c.getFieldValue(inst, col)
 	}
 	insert = insert.Values(values...)
 	_, err = c.DB.InsertTxExt(ctx, c.Table, tx, insert,
@@ -110,10 +233,10 @@ func (c *CrudBase[T]) attemptInsert(ctx context.Context, tx *TXWrapper, inst T,
 	return err
 }
 
-func (c *CrudBase[T]) Upsert(ctx context.Context, inst T, optimization UpsertOptimization, hooks ...PostCompletionHook) (err error) {
+func (c *CrudBase[T]) Upsert(ctx context.Context, inst T, optimization UpsertOptimization, hooks ...PostCompletionHook) (created bool, err error) {
 	ctx, tx, autoCommit, err := c.DB.BeginOrUseTx(ctx)
 	if err != nil {
-		return err
+		return false, err
 	}
 	defer c.DB.RollbackTx(ctx, tx, autoCommit)
 
@@ -124,8 +247,9 @@ func (c *CrudBase[T]) Upsert(ctx context.Context, inst T, optimization UpsertOpt
 	if optimization == UpsertOptimizationNew {
 		opErr := c.attemptInsert(ctx, tx, inst, true /* we want a failure here we can progress past */)
 		optimized = opErr == nil
+		created = optimized
 	} else if optimization == UpsertOptimizationExisting {
-		rowsAffected, opErr := c.attemptReplace(ctx, tx, inst)
+		rowsAffected, opErr := c.updateFromInstance(ctx, tx, inst, true /* full replace */)
 		optimized = opErr == nil && rowsAffected == 1
 	}
 
@@ -137,20 +261,21 @@ func (c *CrudBase[T]) Upsert(ctx context.Context, inst T, optimization UpsertOpt
 				Where(c.idFilter(inst.GetID())),
 		)
 		if err != nil {
-			return err
+			return false, err
 		}
 		existing := msgRows.Next()
 		msgRows.Close()
 
 		if existing {
 			// Replace the existing one
-			if _, err = c.attemptReplace(ctx, tx, inst); err != nil {
-				return err
+			if _, err = c.updateFromInstance(ctx, tx, inst, true /* full replace */); err != nil {
+				return false, err
 			}
 		} else {
 			// Get a useful error out of an insert attempt
+			created = true
 			if err = c.attemptInsert(ctx, tx, inst, false); err != nil {
-				return err
+				return false, err
 			}
 		}
 	}
@@ -159,7 +284,7 @@ func (c *CrudBase[T]) Upsert(ctx context.Context, inst T, optimization UpsertOpt
 		tx.AddPostCommitHook(hook)
 	}
 
-	return c.DB.CommitTx(ctx, tx, autoCommit)
+	return created, c.DB.CommitTx(ctx, tx, autoCommit)
 }
 
 func (c *CrudBase[T]) InsertMany(ctx context.Context, instances []T, allowPartialSuccess bool, hooks ...PostCompletionHook) (err error) {
@@ -174,7 +299,7 @@ func (c *CrudBase[T]) InsertMany(ctx context.Context, instances []T, allowPartia
 		for _, inst := range instances {
 			values := make([]interface{}, len(c.Columns))
 			for i, col := range c.Columns {
-				values[i] = c.GetFieldPtr(inst, col)
+				values[i] = c.getFieldValue(inst, col)
 			}
 			insert = insert.Values(values...)
 		}
@@ -234,7 +359,7 @@ func (c *CrudBase[T]) Replace(ctx context.Context, inst T, hooks ...PostCompleti
 	}
 	defer c.DB.RollbackTx(ctx, tx, autoCommit)
 
-	rowsAffected, err := c.attemptReplace(ctx, tx, inst)
+	rowsAffected, err := c.updateFromInstance(ctx, tx, inst, true /* full replace */)
 	if err != nil {
 		return err
 	} else if rowsAffected < 1 {
@@ -282,14 +407,26 @@ func (c *CrudBase[T]) getReadCols() (tableFrom string, cols, readCols []string)
 	return tableFrom, cols, readCols
 }
 
-func (c *CrudBase[T]) GetByID(ctx context.Context, id *fftypes.UUID) (inst T, err error) {
+func (c *CrudBase[T]) GetByID(ctx context.Context, id *fftypes.UUID, getOpts ...GetOption) (inst T, err error) {
+
+	failNotFound := false
+	for _, o := range getOpts {
+		switch o {
+		case FailIfNotFound:
+			failNotFound = true
+		default:
+			return c.NilValue(), i18n.NewError(ctx, i18n.MsgDBUnknownGetOption, o)
+		}
+	}
+
 	tableFrom, cols, readCols := c.getReadCols()
 	query := sq.Select(readCols...).
 		From(tableFrom).
 		Where(c.idFilter(id))
 	if c.ReadQueryModifier != nil {
 		query = c.ReadQueryModifier(query)
 	}
+
 	rows, _, err := c.DB.Query(ctx, c.Table, query)
 	if err != nil {
 		return c.NilValue(), err
@@ -298,6 +435,9 @@ func (c *CrudBase[T]) GetByID(ctx context.Context, id *fftypes.UUID) (inst T, er
 
 	if !rows.Next() {
 		log.L(ctx).Debugf("%s '%s' not found", c.Table, id)
+		if failNotFound {
+			return c.NilValue(), i18n.NewError(ctx, i18n.Msg404NoResult)
+		}
 		return c.NilValue(), nil
 	}
 
@@ -344,6 +484,59 @@ func (c *CrudBase[T]) Update(ctx context.Context, id *fftypes.UUID, update ffapi
 	}, update, true, hooks...)
 }
 
+// Thanks to the testify/assert package for this
+func isNil(o interface{}) bool {
+	if o == nil {
+		return true
+	}
+
+	containsKind := func(kinds []reflect.Kind, kind reflect.Kind) bool {
+		for i := 0; i < len(kinds); i++ {
+			if kind == kinds[i] {
+				return true
+			}
+		}
+		return false
+	}
+
+	value := reflect.ValueOf(o)
+	kind := value.Kind()
+	isNillableKind := containsKind(
+		[]reflect.Kind{
+			reflect.Chan, reflect.Func,
+			reflect.Interface, reflect.Map,
+			reflect.Ptr, reflect.Slice},
+		kind)
+
+	if isNillableKind && value.IsNil() {
+		return true
+	}
+
+	return false
+}
+
+func (c *CrudBase[T]) UpdateSparse(ctx context.Context, sparseUpdate T, hooks ...PostCompletionHook) (err error) {
+	ctx, tx, autoCommit, err := c.DB.BeginOrUseTx(ctx)
+	if err != nil {
+		return err
+	}
+	defer c.DB.RollbackTx(ctx, tx, autoCommit)
+
+	updateCount, err := c.updateFromInstance(ctx, tx, sparseUpdate, false /* only non-nil fields */)
+	if err != nil {
+		return err
+	}
+	if updateCount < 1 {
+		return i18n.NewError(ctx, i18n.MsgDBNoRowsAffected)
+	}
+
+	for _, hook := range hooks {
+		tx.AddPostCommitHook(hook)
+	}
+
+	return c.DB.CommitTx(ctx, tx, autoCommit)
+}
+
 func (c *CrudBase[T]) UpdateMany(ctx context.Context, filter ffapi.Filter, update ffapi.Update, hooks ...PostCompletionHook) (err error) {
 	return c.attemptUpdate(ctx, func(query sq.UpdateBuilder) (sq.UpdateBuilder, error) {
 		return c.DB.FilterUpdate(ctx, query, filter, c.FilterFieldMap)
@@ -361,6 +554,7 @@ func (c *CrudBase[T]) attemptUpdate(ctx context.Context, filterFn func(sq.Update
 	if err == nil {
 		query, err = filterFn(query)
 	}
+	query = query.Set(ColumnUpdated, fftypes.Now())
 	if err != nil {
 		return err
 	}