Merge staging into master

* Add telemetry on datastore (#56)

* Add debug logs

* Add runtime calculation logging

* add more log

* add datastore telemtry

Co-authored-by: Dylan Tinianov <[email protected]>

* add better telemtry (#58)

* Implement SQL storage (#59)

* implemented sql storage

* remove persistence specific models

* remove unused storage impl

* fix storage interface

* update account controller to changes

* update project controller

* update storage interface

* update scripts controller

* update transactions controller

* update embed controller

* fix project

* init storage

* fix API changes

* test todo

* mod tidy

* move sql to top

* fix models - remove logs

* dissable telemtry

* remove unused fields

* improve init

* fix tests

* fix tests

* add postresql driver

* postgresql client init

* postgresql client init

* update server with const sql

* add host

* add logging

* debug logging

* more debug logs

* more debug logs

* add sentry panic

* clean up code

* update version

* change cors

* Revert "change cors"

This reverts commit fe6c2ea.

* set verison on sentry

* fix update methods

* fix update methods

* clean up code

* clean up code

* clean up stale code (#63)

* Test bugfixes (#65)

* fix factory methods

* add sqlite db to gitignore

* fix test

* fix update

* check and standarize update

* refactor account update

* validate update

* fix missing value

* store init test

* fix execution index

* Cache invalidation and mutex improvements (#70)

* fix factory methods

* add sqlite db to gitignore

* fix test

* fix update

* check and standarize update

* refactor account update

* validate update

* fix missing value

* store init test

* fix execution index

* rename shadowed var

* mutex changes tests

* refactor mutex sync map

* fix test store init

* simplify mutex

* simplify mutex

* refactor cache

* refactor cache

* add eviction handler

* add logger

* bug reproduction

* refactor create init accounts

* track cache eviction

* disable cache test

* added get batch accounts

* optimize get account

* batch get accounts

* log len and cap exe

* add explicit order

* add logging

* add logging

* add logging

* add logging

* clean code

* add extra log for state recreation issue

* fix test

* add missing headers

* State recreation bugfix (#78)

* no version fix

* fix order clause

* added state test

* changed make for pgs store

* temp disable cache

* enable cache

* bugfix recreation failure

* bugfix recreation failure

* added more comments explaning cache

* Emulator pool (#79)

* increase idle connections

* update emulator

* clean up mutex

* get accounts test

* add to cache

* add instance pool

* add tests for instance pool

* integrate instance pool

* create pool

* tidy

* Fix linter (#83)

* Fix linter

* Include CI on staging

* Update linter

* Fix Test_DeployContracts

* Improvement/pool error handling (#80)

* Refill pool when empty

* Fill pool async

* Remove pool refill

* Parallelize test cases (#84)

* Testing/add replica testing (#95)

* Add replica testing

* Found potential cached emulator issue

* Update replica testing

* Update dockerfile

* Create replica tests

* Undo dockerfile change

* temp log

* add version log

* allow all origins

* allow all origins

* origins fix

* test deploy

* update allowed origins

* use env for allowed origins

* explicit order (#96)

* Change cache to thread safe lru (#77)

* Change cache to thread safe lru

* Add cache tests

* Add cache test with executions

* Verify execution block heights

* Fix linter

* Fix linter

* Add error handling to emulatorCache

* Remove cache evicted messages

* Improve cache error handling

* Fix comment

* Update cache error handling

* Update cache error handling

* Update cache error handling

* Add disabled emulator test

* Revert "Parallelize test cases (#84)" (#97)

This reverts commit 91772f0.

Co-authored-by: Dylan Tinianov <[email protected]>

Author: devbugging

.github/workflows/ci.yml

@@ -4,9 +4,9 @@ name: CI
 # events but only for the master branch
 on:
   push:
-    branches: [ master ]
+    branches: [ master, staging ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, staging ]
 
 jobs:
   test:
@@ -17,7 +17,7 @@ jobs:
         fetch-depth: 0
     - uses: actions/setup-go@v2
       with:
-        go-version: '1.18.1'
+        go-version: '1.18.4'
     - uses: actions/cache@v1
       with:
         path: ~/go/pkg/mod
@@ -29,10 +29,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.4'
+      - name: Checkout code
+        uses: actions/checkout@v2
       - name: Run golangci-lint
         uses: golangci/[email protected]
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.47
+          version: v1.49
           args: --timeout=10m

.gitignore

@@ -1,2 +1,3 @@
 .idea
 .DS_Store
+e2e-db

Makefile

@@ -15,12 +15,12 @@ test:
 	GO111MODULE=on go test -v ./...
 
 .PHONY: test-datastore
-test-datastore:
+test-pg:
 	DATASTORE_EMULATOR_HOST=localhost:8081 FLOW_STORAGEBACKEND=datastore GO111MODULE=on go test -v ./...
 
 .PHONY: test-migration
 test-migration:
-	DATASTORE_EMULATOR_HOST=localhost:8081 FLOW_STORAGEBACKEND=datastore GO111MODULE=on go test ./migrate/... -run Test_MigrationV0_12_0
+	DATASTORE_EMULATOR_HOST=localhost:8081 FLOW_STORAGEBACKEND=datastore GO111MODULE=on go test ./migrate/...
 
 
 .PHONY: run
@@ -32,21 +32,19 @@ run:
 	-ldflags "-X github.com/dapperlabs/flow-playground-api/build.version=$(LAST_KNOWN_VERSION)" \
 	server/server.go
 
-.PHONY: run-datastore
-run-datastore:
-	DATASTORE_EMULATOR_HOST=localhost:8081 \
-	FLOW_STORAGEBACKEND=datastore \
-	FLOW_DATASTORE_GCPPROJECTID=flow-developer-playground \
+.PHONY: run-pg
+run-pg:
+	FLOW_DB_USER=postgres \
+	FLOW_DB_PORT=5432 \
+	FLOW_DB_NAME=dapper \
+	FLOW_DB_HOST=localhost \
+	FLOW_STORAGEBACKEND=postgresql \
 	FLOW_DEBUG=true FLOW_SESSIONCOOKIESSECURE=false \
 	GO111MODULE=on \
 	go run \
 	-ldflags "-X github.com/dapperlabs/flow-playground-api/build.version=$(LAST_KNOWN_VERSION)" \
 	server/server.go
 
-.PHONY: start-datastore-emulator
-start-datastore-emulator:
-	gcloud beta emulators datastore start --no-store-on-disk
-
 .PHONY: ci
 ci: check-tidy test check-headers
 

adapter/state.go

@@ -29,7 +29,7 @@ type accountState struct {
 // stateToAPI removes any state values that are blockchain system values and not relevant to user usage of the playground.
 func stateToAPI(state string) string {
 	if state == "" {
-		return state
+		return "{}"
 	}
 
 	var accState accountState
@@ -41,8 +41,7 @@ func stateToAPI(state string) string {
 
 	// return empty as empty object, no keys - v0 adapter
 	if len(accState.Public)+len(accState.Private)+len(accState.Storage) == 0 {
-		emptyState, _ := json.Marshal("{}")
-		return string(emptyState)
+		return "{}"
 	}
 
 	adaptedState, _ := json.Marshal(accState)

auth/auth.go

@@ -20,8 +20,6 @@ package auth
 
 import (
 	"context"
-	"github.com/dapperlabs/flow-playground-api/telemetry"
-
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
@@ -90,10 +88,7 @@ func (a *Authenticator) GetOrCreateUser(ctx context.Context) (*model.User, error
 //
 // This function checks for access using both the new and legacy authentication schemes. If
 // a user has legacy access, their authentication is then migrated to use the new scheme.
-func (a *Authenticator) CheckProjectAccess(ctx context.Context, proj *model.InternalProject) error {
-	telemetry.StartRuntimeCalculation()
-	defer telemetry.EndRuntimeCalculation()
-	telemetry.DebugLog("[auth] Check Project Access")
+func (a *Authenticator) CheckProjectAccess(ctx context.Context, proj *model.Project) error {
 	var user *model.User
 	var err error
 
@@ -107,7 +102,6 @@ func (a *Authenticator) CheckProjectAccess(ctx context.Context, proj *model.Inte
 	}
 
 	if a.hasProjectAccess(user, proj) {
-		telemetry.DebugLog("[auth] Check Project Access")
 		err = sessions.Save(ctx, session)
 		if err != nil {
 			return errors.New("access denied")
@@ -136,10 +130,7 @@ func (a *Authenticator) CheckProjectAccess(ctx context.Context, proj *model.Inte
 }
 
 func (a *Authenticator) getCurrentUser(userIDStr string) (*model.User, error) {
-	telemetry.StartRuntimeCalculation()
-	defer telemetry.EndRuntimeCalculation()
 	var user model.User
-
 	var userID uuid.UUID
 
 	err := userID.UnmarshalText([]byte(userIDStr))
@@ -155,15 +146,15 @@ func (a *Authenticator) getCurrentUser(userIDStr string) (*model.User, error) {
 	return &user, nil
 }
 
-func (a *Authenticator) hasProjectAccess(user *model.User, proj *model.InternalProject) bool {
+func (a *Authenticator) hasProjectAccess(user *model.User, proj *model.Project) bool {
 	return user != nil && proj.IsOwnedBy(user.ID)
 }
 
-func (a *Authenticator) hasLegacyProjectAccess(ctx context.Context, proj *model.InternalProject) bool {
+func (a *Authenticator) hasLegacyProjectAccess(ctx context.Context, proj *model.Project) bool {
 	return legacyauth.ProjectInSession(ctx, proj)
 }
 
-func (a *Authenticator) migrateLegacyProjectAccess(user *model.User, proj *model.InternalProject) (*model.User, error) {
+func (a *Authenticator) migrateLegacyProjectAccess(user *model.User, proj *model.Project) (*model.User, error) {
 	var err error
 
 	if user == nil {
@@ -188,7 +179,7 @@ func (a *Authenticator) createNewUser() (*model.User, error) {
 
 	err := a.store.InsertUser(user)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "could not insert the user")
 	}
 
 	return user, nil

auth/legacy/auth.go

@@ -36,7 +36,7 @@ const projectSecretKeyName = "project-secret"
 //
 // A project is authorized in a session if the session contains a reference to the
 // project's secret.
-func ProjectInSession(ctx context.Context, proj *model.InternalProject) bool {
+func ProjectInSession(ctx context.Context, proj *model.Project) bool {
 	session := sessions.Get(ctx, getProjectSessionName(proj))
 
 	secret, ok := session.Values[projectSecretKeyName]
@@ -52,7 +52,7 @@ func ProjectInSession(ctx context.Context, proj *model.InternalProject) bool {
 	return secretStr == proj.Secret.String()
 }
 
-func getProjectSessionName(proj *model.InternalProject) string {
+func getProjectSessionName(proj *model.Project) string {
 	return getProjectSessionNameFromString(proj.ID.String())
 }
 

blockchain/cache.go

@@ -19,69 +19,61 @@
 package blockchain
 
 import (
-	"fmt"
-	"github.com/dapperlabs/flow-playground-api/model"
-	"github.com/dapperlabs/flow-playground-api/telemetry"
 	"github.com/getsentry/sentry-go"
-	"github.com/golang/groupcache/lru"
 	"github.com/google/uuid"
-	"github.com/pkg/errors"
+	lru "github.com/hashicorp/golang-lru"
 )
 
-// newCache returns a new instance of cache with provided capacity.
-func newCache(capacity int) *cache {
-	return &cache{
-		cache: lru.New(capacity),
-	}
+// emulatorCache caches the emulator state.
+//
+// In the environment where multiple replicas maintain its own cache copy it can get into multiple states:
+// - it can get stale because replica A receives transaction execution 1, and replica B receives transaction execution 2,
+//   then replica A needs to apply missed transaction execution 2 before continuing
+// - it can be outdated because replica A receives project reset, which clears all executions and the cache, but replica B
+//   doesn't receive that request so on next run it receives 0 executions but cached emulator contains state from previous
+//   executions that wasn't cleared
+type emulatorCache struct {
+	cache *lru.Cache
 }
 
-type cache struct {
-	cache *lru.Cache
+// newEmulatorCache returns a new instance of emulatorCache with provided capacity.
+func newEmulatorCache(capacity int) *emulatorCache {
+	cache, err := lru.New(capacity)
+	if err != nil {
+		sentry.CaptureMessage("Continuing without emulator caching: " + err.Error())
+	}
+
+	return &emulatorCache{
+		cache: cache,
+	}
 }
 
-// reset the cache for the ID.
-func (c *cache) reset(ID uuid.UUID) {
+// reset the cached emulator for the ID.
+func (c *emulatorCache) reset(ID uuid.UUID) {
+	if c.cache == nil {
+		return
+	}
 	c.cache.Remove(ID)
 }
 
-// get returns a cached emulator if exists, but also checks if it's stale.
-//
-// based on the executions the function receives it compares that to the emulator block height, since
-// one execution is always one block it can compare the heights to the length. If it finds some executions
-// that are not part of emulator it returns that subset, so they can be applied on top.
-func (c *cache) get(
-	ID uuid.UUID,
-	executions []*model.TransactionExecution,
-) (blockchain, []*model.TransactionExecution, error) {
-	telemetry.DebugLog("[cache] get - start get executions from ID")
+// get returns a cached emulator for specified ID if it exists
+func (c *emulatorCache) get(ID uuid.UUID) *emulator {
+	if c.cache == nil {
+		return nil
+	}
 
 	val, ok := c.cache.Get(ID)
 	if !ok {
-		return nil, executions, nil
-	}
-
-	emulator := val.(blockchain)
-	latest, err := emulator.getLatestBlock()
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "cache failure")
+		return nil
 	}
 
-	// this should never happen, sanity check
-	if int(latest.Header.Height) > len(executions) {
-		err := fmt.Errorf("cache failure, block height is higher than executions count")
-		sentry.CaptureException(err)
-		return nil, nil, err
-	}
-
-	telemetry.DebugLog("[cache] get - got latest block from emulator")
-
-	// this will return only executions that are missing from the emulator
-	return emulator, executions[latest.Header.Height:], nil
+	return val.(*emulator)
 }
 
-// add new entry in the cache.
-func (c *cache) add(ID uuid.UUID, emulator blockchain) {
-	telemetry.DebugLog("[cache] add - start add emulator to cache")
+// add new emulator to the cache.
+func (c *emulatorCache) add(ID uuid.UUID, emulator *emulator) {
+	if c.cache == nil {
+		return
+	}
 	c.cache.Add(ID, emulator)
-	telemetry.DebugLog("[cache] add - added emulator to cache")
 }