Merge pull request #1116 from onetechnical/onetechnical/beta2.0.14

Algorand Release 2.0.14-beta

Author: algojohnlee

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,9 +15,8 @@ It is also a good idea to run tests:
 
 ## Summary
 
-Explain the goal of this change and what problem it is solving.
+Explain the goal of this change and what problem it is solving. Format this cleanly so that it may be used for a commit message, as your changes will be squash-merged.
 
 ## Test Plan
 
 How did you test these changes? Please provide the exact scenarios you tested in as much detail as possible including commands, output and rationale.
-

.travis.yml

@@ -166,7 +166,7 @@ addons:
   artifacts:
     s3_region: "us-east-1"
     paths:
-      - $(git ls-files -o | grep -v crypto/libsodium-fork | grep -v crypto/lib/ | grep -v ^gen/ | grep -v swagger.json.validated | tr "\n" ":")
+      - $(git ls-files -o | grep -v crypto/libs | grep -v crypto/copies | grep -v crypto/libsodium-fork | grep -v crypto/lib/ | grep -v ^gen/ | grep -v swagger.json.validated | tr "\n" ":")
 
 notifications:
   slack:

CONTRIBUTING.md

@@ -2,7 +2,18 @@
 
 If you are interested in contributing to the project, we welcome and thank you. We want to make the best decentralized and effective blockchain platform available and we appreciate your willingness to help us.
 
-The algorand project is composed of several repositories on GitHub. Specifically we have repositories for the main node software,the JavaScript SDK, the Go SDK, and  the Java SDK. Additionally we have repositories for projects we forked like go-codec that make up some of the main dependencies for a node. Each of these will follow similar guidelines on how to contribute. The main difference between these concerns code formatting and deploying changes to production.
+The [Algorand GitHub Organization](https://github.com/algorand) has all of our open source projects, and dependencies which we fork and use in those projects. This contribution guide applies to all of these.
+
+Some of our most active projects include:
+* [go-algorand](https://github.com/algorand/go-algorand) - Algorand node software (this repository)
+* [go-algorand-sdk](https://github.com/algorand/go-algorand-sdk) - Golang SDK
+* [java-algorand-sdk](https://github.com/algorand/java-algorand-sdk) - Java SDK
+* [js-algorand-sdk](https://github.com/algorand/js-algorand-sdk) - JavaScript SDK
+* [indexer](https://github.com/algorand/indexer) - Blockchain analytics database
+* [ledger-app-algorand](https://github.com/algorand/ledger-app-algorand) - Ledger hardware wallet application
+* [mule](https://github.com/algorand/mule) - Continuous Integration automation tool
+* [py-algorand-sdk](https://github.com/algorand/py-algorand-sdk) - Python SDK
+* [sandbox](https://github.com/algorand/sandbox) - Algorand node quickstart tool
 
 # Filing Issues
 
@@ -24,7 +35,9 @@ If you have any questions, don't hesitate to contact us at [email protected]
 
 # Contribution Model
 
-For each of our repositories we use the same model for contributing code. Developers wanting to contribute must create pull requests. This process is described in the GitHub [Creating a pull request from a fork](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) documentation. Each pull request should be initiated against the master branch in the Algorand repository.  After a pull request is submitted the core development team will review the submission and communicate with the developer using the comments sections of the PR. After the submission is reviewed and approved, it will be merged into the master branch of the source. These changes will be merged to our release branch on the next viable release date. For the SDKs, this may be immediate. Changes to the node software may take more time as we must ensure and verify the security, as well as apply protocol upgrades in an orderly way.
+For each of our repositories we use the same model for contributing code. Developers wanting to contribute must create pull requests. This process is described in the GitHub [Creating a pull request from a fork](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) documentation. Each pull request should be initiated against the `master` branch in the Algorand repository.  After a pull request is submitted the core development team will review the submission and communicate with the developer using the comments sections of the PR. After the submission is reviewed and approved, it will be merged into the `master` branch of the source. These changes will be merged to our release branch on the next viable release date. For the SDKs, this may be immediate. Changes to the node software may take more time as we must ensure and verify the security, as well as apply protocol upgrades in an orderly way.
+
+Note: some of our projects are using gitflow, for these you will open pull requests against the `develop` branch.
 
 Again, if you have a patch for a critical security vulnerability, please use our [vulnerability disclosure form][vuln_url] instead of creating a PR.  We'll follow up with you on distributing the patch before we merge it.
 

Makefile

@@ -1,8 +1,8 @@
-GOPATH		:= $(shell go env GOPATH)
+export GOPATH		:= $(shell go env GOPATH)
 GOPATH1		:= $(firstword $(subst :, ,$(GOPATH)))
-export GOPATH
-GO111MODULE	:= on
-export GO111MODULE
+export GO111MODULE	:= on
+export GOPROXY := https://gocenter.io
+
 UNAME		:= $(shell uname)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
@@ -52,14 +52,15 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node
+MSGP_GENERATE	:= ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger
 
 default: build
 
 # tools
 
 fmt:
 	go fmt ./...
+	./scripts/check_license.sh -i
 
 fix: build
 	$(GOPATH1)/bin/algofix */
@@ -73,13 +74,10 @@ lint: deps
 vet:
 	go vet ./...
 
-check_license:
-	./scripts/check_license.sh
-
 check_shell:
 	find . -type f -name "*.sh" -exec shellcheck {} +
 
-sanity: vet fix lint fmt check_license
+sanity: vet fix lint fmt
 
 cover:
 	go test $(GOTAGS) -coverprofile=cover.out $(UNIT_TEST_SOURCES)
@@ -276,14 +274,8 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt vet lint check_license check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN
-
-### TARGETS FOR CICD PROCESS
+.PHONY: default fmt vet lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN
 
-ci-deps:
-	scripts/configure_dev-deps.sh && \
-	scripts/check_deps.sh
+###### TARGETS FOR CICD PROCESS ######
+include ./scripts/release/mule/Makefile.mule
 
-ci-build: buildsrc gen
-	mkdir -p $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) && \
-	PKG_ROOT=$(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) NO_BUILD=True VARIATIONS=$(OS_TYPE)/$(ARCH) scripts/build_packages.sh $(OS_TYPE)/$(ARCH)

THANKS.md

@@ -1,7 +1,7 @@
 
 ## Thanks
 
-A big thank you to everyone who has contributed to the `go-algorand` codebase. 
+A big thank you to everyone who has contributed to the `go-algorand` codebase.
 
 ### External Contributors
 In no particular order:
@@ -11,6 +11,7 @@ In no particular order:
 - scnale
 - jsign
 - RomitKumar
+- jeapostrophe
 
 ### Bug Reports
 - Nanyan

agreement/abstractions.go

@@ -276,6 +276,10 @@ type Network interface {
 	// Disconnect sends the Network a hint to disconnect to the peer
 	// associated with the given MessageHandle.
 	Disconnect(MessageHandle)
+
+	// Start notifies the network that the agreement service is ready
+	// to start receiving messages.
+	Start()
 }
 
 // RandomSource is an abstraction over the random number generator.

agreement/cadaver.go

@@ -119,14 +119,21 @@ func (c *cadaver) trySetup() bool {
 		c.out.Close()
 		err := os.Rename(c.filename(), c.filename()+".archive")
 		if err != nil {
-			logging.Base().Error(err)
-			c.failed = err
-			return false
+			if os.IsNotExist(err) {
+				// we can't rename the cadaver file since it doesn't exists.
+				// this typically happens when it being externally deleted, and could happen
+				// far before we close the handle above.
+				logging.Base().Info(err)
+			} else {
+				logging.Base().Warn(err)
+				c.failed = err
+				return false
+			}
 		}
 
 		err = c.init()
 		if err != nil {
-			logging.Base().Error(err)
+			logging.Base().Warn(err)
 			c.failed = err
 			return false
 		}

agreement/demux.go

@@ -66,24 +66,35 @@ type demux struct {
 	log logging.Logger
 }
 
+// demuxParams contains the parameters required to initliaze a new demux object
+type demuxParams struct {
+	net               Network
+	ledger            LedgerReader
+	validator         BlockValidator
+	voteVerifier      *AsyncVoteVerifier
+	processingMonitor EventsProcessingMonitor
+	log               logging.Logger
+	monitor           *coserviceMonitor
+}
+
 // makeDemux initializes the goroutines needed to process external events, setting up the appropriate channels.
 //
 // It must be called before other methods are called.
-func makeDemux(net Network, ledger LedgerReader, validator BlockValidator, voteVerifier *AsyncVoteVerifier, processingMonitor EventsProcessingMonitor, log logging.Logger) (d *demux) {
+func makeDemux(params demuxParams) (d *demux) {
 	d = new(demux)
-	d.crypto = makeCryptoVerifier(ledger, validator, voteVerifier, log)
-	d.log = log
-	d.ledger = ledger
+	d.crypto = makeCryptoVerifier(params.ledger, params.validator, params.voteVerifier, params.log)
+	d.log = params.log
+	d.ledger = params.ledger
+	d.monitor = params.monitor
+	d.queue = make([]<-chan externalEvent, 0)
+	d.processingMonitor = params.processingMonitor
 
 	tokenizerCtx, cancelTokenizers := context.WithCancel(context.Background())
-	d.rawVotes = d.tokenizeMessages(tokenizerCtx, net, protocol.AgreementVoteTag, decodeVote)
-	d.rawProposals = d.tokenizeMessages(tokenizerCtx, net, protocol.ProposalPayloadTag, decodeProposal)
-	d.rawBundles = d.tokenizeMessages(tokenizerCtx, net, protocol.VoteBundleTag, decodeBundle)
+	d.rawVotes = d.tokenizeMessages(tokenizerCtx, params.net, protocol.AgreementVoteTag, decodeVote)
+	d.rawProposals = d.tokenizeMessages(tokenizerCtx, params.net, protocol.ProposalPayloadTag, decodeProposal)
+	d.rawBundles = d.tokenizeMessages(tokenizerCtx, params.net, protocol.VoteBundleTag, decodeBundle)
 	d.cancelTokenizers = cancelTokenizers
 
-	d.queue = make([]<-chan externalEvent, 0)
-	d.processingMonitor = processingMonitor
-
 	return d
 }
 

agreement/fuzzer/networkFacade_test.go

@@ -77,7 +77,7 @@ func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade {
 	n := &NetworkFacade{
 		fuzzer:         fuzzer,
 		nodeID:         nodeID,
-		mux:            network.MakeMultiplexer(),
+		mux:            network.MakeMultiplexer(fuzzer.log),
 		clocks:         make(map[int]chan time.Time),
 		eventsQueues:   make(map[string]int),
 		eventsQueuesCh: make(chan int, 1000),
@@ -258,7 +258,7 @@ func (n *NetworkFacade) RegisterHandlers(dispatch []network.TaggedMessageHandler
 
 // ClearHandlers
 func (n *NetworkFacade) ClearHandlers() {
-	n.mux.ClearHandlers()
+	n.mux.ClearHandlers([]network.Tag{})
 }
 
 // SetDownstreamFilter sets the downstream filter.

agreement/gossip/network.go

@@ -63,13 +63,16 @@ func WrapNetwork(net network.GossipNode, log logging.Logger) agreement.Network {
 	i.net = net
 	i.log = log
 
+	return i
+}
+
+func (i *networkImpl) Start() {
 	handlers := []network.TaggedMessageHandler{
 		{Tag: protocol.AgreementVoteTag, MessageHandler: network.HandlerFunc(i.processVoteMessage)},
 		{Tag: protocol.ProposalPayloadTag, MessageHandler: network.HandlerFunc(i.processProposalMessage)},
 		{Tag: protocol.VoteBundleTag, MessageHandler: network.HandlerFunc(i.processBundleMessage)},
 	}
-	net.RegisterHandlers(handlers)
-	return i
+	i.net.RegisterHandlers(handlers)
 }
 
 func messageMetadataFromHandle(h agreement.MessageHandle) *messageMetadata {

agreement/gossip/networkFull_test.go

@@ -81,6 +81,7 @@ func spinNetwork(t *testing.T, nodesCount int) ([]*networkImpl, []*messageCounte
 	for _, gossipNode := range gossipNodes {
 		networkImpl := WrapNetwork(gossipNode, log).(*networkImpl)
 		networkImpls = append(networkImpls, networkImpl)
+		networkImpl.Start()
 		msgCounter := startMessageCounter(networkImpl)
 		msgCounters = append(msgCounters, msgCounter)
 	}

agreement/gossip/network_test.go

@@ -45,6 +45,7 @@ type whiteholeDomain struct {
 	messagesMu   deadlock.Mutex
 	messagesCond *sync.Cond
 	peerIdx      uint32
+	log          logging.Logger
 }
 
 type whiteholeNetwork struct {
@@ -55,6 +56,7 @@ type whiteholeNetwork struct {
 	quit         chan struct{}
 	domain       *whiteholeDomain
 	disconnected map[uint32]bool
+	log          logging.Logger
 }
 
 func (d *whiteholeDomain) syncNetwork(networks ...*whiteholeNetwork) {
@@ -89,7 +91,7 @@ func (w *whiteholeNetwork) RegisterHandlers(dispatch []network.TaggedMessageHand
 
 // ClearHandlers deregisters all the existing message handlers.
 func (w *whiteholeNetwork) ClearHandlers() {
-	w.mux.ClearHandlers()
+	w.mux.ClearHandlers([]network.Tag{})
 }
 
 func (w *whiteholeNetwork) Address() (string, bool) {
@@ -315,7 +317,7 @@ func makewhiteholeNetwork(domain *whiteholeDomain) *whiteholeNetwork {
 	w := &whiteholeNetwork{
 		peer:         atomic.AddUint32(&domain.peerIdx, 1),
 		lastMsgRead:  uint32(len(domain.messages)),
-		mux:          network.MakeMultiplexer(),
+		mux:          network.MakeMultiplexer(domain.log),
 		domain:       domain,
 		disconnected: make(map[uint32]bool),
 	}
@@ -327,6 +329,7 @@ func spinNetworkImpl(domain *whiteholeDomain) (whiteholeNet *whiteholeNetwork, c
 	netImpl := WrapNetwork(whiteholeNet, logging.Base()).(*networkImpl)
 	counter = startMessageCounter(netImpl)
 	whiteholeNet.Start()
+	netImpl.Start()
 	return
 }
 
@@ -336,6 +339,7 @@ func TestNetworkImpl(t *testing.T) {
 	domain := &whiteholeDomain{
 		messages: make([]sentMessage, 0),
 		peerIdx:  uint32(0),
+		log:      logging.TestingLog(t),
 	}
 	domain.messagesCond = sync.NewCond(&domain.messagesMu)
 

agreement/pseudonode.go

@@ -118,19 +118,31 @@ type pseudonodeVerifier struct {
 //msgp:ignore verifiedCryptoResults
 type verifiedCryptoResults []asyncVerifyVoteResponse
 
-func makePseudonode(factory BlockFactory, validator BlockValidator, keys KeyManager, ledger Ledger, voteVerifier *AsyncVoteVerifier, log serviceLogger) pseudonode {
+// pseudonodeParams struct provide the parameters required to create a pseudonode
+type pseudonodeParams struct {
+	factory      BlockFactory
+	validator    BlockValidator
+	keys         KeyManager
+	ledger       Ledger
+	voteVerifier *AsyncVoteVerifier
+	log          serviceLogger
+	monitor      *coserviceMonitor
+}
+
+func makePseudonode(params pseudonodeParams) pseudonode {
 	pn := asyncPseudonode{
-		factory:   factory,
-		validator: validator,
-		keys:      keys,
-		ledger:    ledger,
-		log:       log,
+		factory:   params.factory,
+		validator: params.validator,
+		keys:      params.keys,
+		ledger:    params.ledger,
+		log:       params.log,
 		quit:      make(chan struct{}),
 		closeWg:   &sync.WaitGroup{},
+		monitor:   params.monitor,
 	}
 
-	pn.proposalsVerifier = pn.makePseudonodeVerifier(voteVerifier)
-	pn.votesVerifier = pn.makePseudonodeVerifier(voteVerifier)
+	pn.proposalsVerifier = pn.makePseudonodeVerifier(params.voteVerifier)
+	pn.votesVerifier = pn.makePseudonodeVerifier(params.voteVerifier)
 	return pn
 }
 

agreement/pseudonode_test.go

@@ -141,7 +141,15 @@ func TestPseudonode(t *testing.T) {
 	sLogger := serviceLogger{logging.Base()}
 
 	keyManager := simpleKeyManager(accounts)
-	pb := makePseudonode(testBlockFactory{Owner: 0}, testBlockValidator{}, keyManager, ledger, MakeAsyncVoteVerifier(nil), sLogger)
+	pb := makePseudonode(pseudonodeParams{
+		factory:      testBlockFactory{Owner: 0},
+		validator:    testBlockValidator{},
+		keys:         keyManager,
+		ledger:       ledger,
+		voteVerifier: MakeAsyncVoteVerifier(nil),
+		log:          sLogger,
+		monitor:      nil,
+	})
 	defer pb.Quit()
 	spn := makeSerializedPseudonode(testBlockFactory{Owner: 0}, testBlockValidator{}, keyManager, ledger)
 	defer spn.Quit()