Merge pull request #1347 from onetechnical/onetechnical/relbeta2.1.0

Onetechnical/relbeta2.1.0

Author: algojohnlee

.gitignore

@@ -61,3 +61,5 @@ data/transactions/logic/*.md
 # Folder for collecting release assets
 assets
 
+# test binaries
+*.test

.travis.yml

@@ -3,7 +3,6 @@ go:
   - "1.12"
 go_import_path: github.com/algorand/go-algorand
 language: go
-sudo: required
 
 # Don't build tags
 if: tag IS blank

Makefile

@@ -1,18 +1,19 @@
 export GOPATH		:= $(shell go env GOPATH)
 GOPATH1		:= $(firstword $(subst :, ,$(GOPATH)))
 export GO111MODULE	:= on
-export GOPROXY := https://gocenter.io
+export GOPROXY := direct
 
 UNAME		:= $(shell uname)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
+S3_RELEASE_BUCKET = $$S3_RELEASE_BUCKET
 
 # If build number already set, use it - to ensure same build number across multiple platforms being built
 BUILDNUMBER      ?= $(shell ./scripts/compute_build_number.sh)
 COMMITHASH       := $(shell ./scripts/compute_build_commit.sh)
-BUILDBRANCH      ?= $(shell ./scripts/compute_branch.sh)
-BUILDCHANNEL     ?= $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
+BUILDBRANCH      := $(shell ./scripts/compute_branch.sh)
+CHANNEL          ?= $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
 DEFAULTNETWORK   ?= $(shell ./scripts/compute_branch_network.sh $(BUILDBRANCH))
 DEFAULT_DEADLOCK ?= $(shell ./scripts/compute_branch_deadlock_default.sh $(BUILDBRANCH))
 
@@ -47,7 +48,7 @@ GOLDFLAGS_BASE  := -X github.com/algorand/go-algorand/config.BuildNumber=$(BUILD
 		 -extldflags \"$(EXTLDFLAGS)\"
 
 GOLDFLAGS := $(GOLDFLAGS_BASE) \
-		 -X github.com/algorand/go-algorand/config.Channel=$(BUILDCHANNEL)
+		 -X github.com/algorand/go-algorand/config.Channel=$(CHANNEL)
 
 UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... ))
@@ -155,8 +156,14 @@ $(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).valid
 
 build: buildsrc gen
 
+# We're making an empty file in the go-cache dir to
+# get around a bug in go build where it will fail
+# to cache binaries from time to time on empty NFS
+# dirs
 buildsrc: crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	mkdir -p tmp/go-cache && \
+	touch tmp/go-cache/file.txt && \
+	GOCACHE=$(SRCPATH)/tmp/go-cache go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 SOURCES_RACE := github.com/algorand/go-algorand/cmd/kmd
 
@@ -190,13 +197,13 @@ test: build
 
 fulltest: build-race
 	for PACKAGE_DIRECTORY in $(UNIT_TEST_SOURCES) ; do \
-		go test $(GOTAGS) -timeout 2000s -race $$PACKAGE_DIRECTORY; \
+		go test $(GOTAGS) -timeout 2500s -race $$PACKAGE_DIRECTORY; \
 	done
 
 shorttest: build-race $(addprefix short_test_target_, $(UNIT_TEST_SOURCES))
 
 $(addprefix short_test_target_, $(UNIT_TEST_SOURCES)): build
-	@go test $(GOTAGS) -short -timeout 2000s -race $(subst short_test_target_,,$@)
+	@go test $(GOTAGS) -short -timeout 2500s -race $(subst short_test_target_,,$@)
 
 integration: build-race
 	./test/scripts/run_integration_tests.sh
@@ -233,7 +240,10 @@ node_exporter: $(GOPATH1)/bin/node_exporter
 # The file is was taken from the S3 cloud and it traditionally stored at
 # /travis-build-artifacts-us-ea-1.algorand.network/algorand/node_exporter/latest/node_exporter-stable-linux-x86_64.tar.gz
 $(GOPATH1)/bin/node_exporter:
-	tar -xzvf installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz -C $(GOPATH1)/bin
+	mkdir -p $(GOPATH1)/bin && \
+	cd $(GOPATH1)/bin && \
+	tar -xzvf $(SRCPATH)/installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz && \
+	cd -
 
 # deploy
 
@@ -279,3 +289,10 @@ install: build
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule
 
+SUPPORTED_ARCHIVE_OS_ARCH = linux/amd64 linux/arm64 linux/arm darwin/amd64
+
+archive:
+	CHANNEL=$(CHANNEL) \
+	PATH=$(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/bin:$${PATH} \
+	scripts/upload_version.sh $(CHANNEL) $(SRCPATH)/tmp/node_pkgs $(S3_RELEASE_BUCKET)
+

README.md

@@ -14,8 +14,8 @@ about using and installing the algorand platform.
 
 ## Building from source ##
 
-Development is done using the [Go Programming Language](https://golang.org/)
-version 1.12.x, and this document assumes that you have a functioning
+Development is done using the [Go Programming Language](https://golang.org/).
+The version of go is specified in the project's [go.mod](go.mod) file. This document assumes that you have a functioning
 environment setup. If you need assistance setting up an environment please visit
 the [official Go documentation website](https://golang.org/doc/).
 

agreement/msgp_gen.go

@@ -16,7 +16,6 @@
 
 package agreement
 
-//go:generate dbgen -i agree.sql -p agreement -n agree -o agreeInstall.go
 import (
 	"context"
 	"database/sql"
@@ -74,7 +73,7 @@ func persist(log serviceLogger, crash db.Accessor, Round basics.Round, Period pe
 		log.with(logEvent).Info("persisted state to the database")
 	}()
 
-	err = crash.Atomic(func(tx *sql.Tx) error {
+	err = crash.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		_, err := tx.Exec("insert or replace into Service (rowid, data) values (1, ?)", raw)
 		return err
 	})
@@ -92,7 +91,7 @@ func persist(log serviceLogger, crash db.Accessor, Round basics.Round, Period pe
 func reset(log logging.Logger, crash db.Accessor) (err error) {
 	logging.Base().Infof("reset (agreement): resetting crash state")
 
-	err = crash.Atomic(func(tx *sql.Tx) (res error) {
+	err = crash.Atomic(func(ctx context.Context, tx *sql.Tx) (res error) {
 		// we could not retrieve our state, so wipe it
 		_, err := tx.Exec("delete from Service")
 		if err != nil {
@@ -115,11 +114,11 @@ func restore(log logging.Logger, crash db.Accessor) (raw []byte, err error) {
 		}
 	}()
 
-	crash.Atomic(func(tx *sql.Tx) error {
+	crash.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		return agreeInstallDatabase(tx)
 	}) // ignore error
 
-	err = crash.Atomic(func(tx *sql.Tx) (res error) {
+	err = crash.Atomic(func(ctx context.Context, tx *sql.Tx) (res error) {
 		var reset bool
 		defer func() {
 			if !reset {

agreement/msgp_gen_test.go

@@ -17,6 +17,7 @@
 package agreement
 
 import (
+	"context"
 	"database/sql"
 	"testing"
 	"time"
@@ -85,7 +86,7 @@ func TestAgreementPersistence(t *testing.T) {
 	require.NoError(t, err)
 	defer accessor.Close()
 
-	accessor.Atomic(func(tx *sql.Tx) error {
+	accessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		return agreeInstallDatabase(tx)
 	}) // ignore error
 
@@ -112,7 +113,7 @@ func BenchmarkAgreementPersistence(b *testing.B) {
 	accessor, _ := db.MakeAccessor(b.Name()+"_crash.db", false, true)
 	defer accessor.Close()
 
-	accessor.Atomic(func(tx *sql.Tx) error {
+	accessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		return agreeInstallDatabase(tx)
 	}) // ignore error
 
@@ -138,7 +139,7 @@ func BenchmarkAgreementPersistenceRecovery(b *testing.B) {
 	accessor, _ := db.MakeAccessor(b.Name()+"_crash.db", false, true)
 	defer accessor.Close()
 
-	accessor.Atomic(func(tx *sql.Tx) error {
+	accessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		return agreeInstallDatabase(tx)
 	}) // ignore error
 

agreement/persistence.go

@@ -24,20 +24,10 @@ import (
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
-	"github.com/algorand/go-algorand/data/committee" //TODO(upgrade) remove this line
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 )
 
-// TODO(upgrade) remove the entire lessMaybeBuggy function once the upgrade goes through
-func lessMaybeBuggy(cred, other committee.Credential) bool {
-	// this function calls either Less or LessBuggy depending on ConsensusCurrentVersion, which is what the agreement tests use
-	if config.Consensus[protocol.ConsensusCurrentVersion].UseBuggyProposalLowestOutput {
-		return cred.LessBuggy(other)
-	}
-	return cred.Less(other)
-}
-
 var playerTracer tracer
 
 func init() {
@@ -69,8 +59,7 @@ func generateProposalEvents(t *testing.T, player player, accs testAccountData, f
 	lowestCredential := votes[0].Cred
 	lowestProposal = votes[0].R.Proposal
 	for _, vote := range votes {
-		// if vote.Cred.Less(lowestCredential) { //TODO(upgrade) uncomment this line
-		if lessMaybeBuggy(vote.Cred, lowestCredential) { // TODO(upgrade) remove this line
+		if vote.Cred.Less(lowestCredential) {
 			lowestCredential = vote.Cred
 			lowestProposal = vote.R.Proposal
 		}