Merge pull request #1681 from onetechnical/onetechnical/relstable2.2.0

Onetechnical/relstable2.2.0

Author: algojohnlee

.github/ISSUE_TEMPLATE/algorand-engineering-team-issue-template.md

@@ -0,0 +1,18 @@
+---
+name: Algorand Engineering Team Issue Template
+about: This is the template that Algorand internal team members use in conjunction
+  with their task management process. Feel free to use if you're an external contributor.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Summary
+*Describe the problem identified or the general goal of this issue*
+
+## Scope/Requirements
+*What's involved in this issue? What's required to achieve the goal?*
+
+## Urgency/Relative Priority
+*How urgent is this issue? What are the timing considerations to take into account?*

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,9 +1,12 @@
 ---
-name: '🐜 Bug report'
-about: 'Report a reproducible bug.'
+name: "\U0001F41C Bug report"
+about: Report a reproducible bug.
 title: ''
-labels: 'new-bug'
+labels: new-bug
+assignees: ''
+
 ---
+
 <!--
 NOTE: If this issue relates to security, please use the vulnerability disclosure form here:
 https://www.algorand.com/resources/blog/security

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,9 +1,12 @@
 ---
-name: '🔔 Feature Request'
-about: 'Suggestions for how we can improve the algorand platform.'
+name: "\U0001F514 Feature Request"
+about: Suggestions for how we can improve the algorand platform.
 title: ''
-labels: 'new-feature-request'
+labels: new-feature-request
+assignees: ''
+
 ---
+
 <!--
 NOTE: If this issue relates to security, please use the vulnerability disclosure form here:
 https://www.algorand.com/resources/blog/security

.github/ISSUE_TEMPLATE/question.md

@@ -1,8 +1,10 @@
 ---
-name: '❓ Question'
-about: 'General questions related to the algorand platform.'
+name: "❓ Question"
+about: General questions related to the algorand platform.
 title: ''
-labels: 'question'
+labels: question
+assignees: ''
+
 ---
 
 🚨 The issue tracker is not for questions. 🚨

.gitignore

@@ -63,3 +63,6 @@ assets
 
 # test binaries
 *.test
+
+index.html
+

.travis.yml

@@ -1,3 +1,4 @@
+os: linux
 dist: bionic
 go_import_path: github.com/algorand/go-algorand
 language: go
@@ -76,6 +77,15 @@ jobs:
       name: MacOS AMD64 Integration Test
       script:
         - ./scripts/travis/integration_test.sh
+    - # same stage, parallel job
+       os: windows
+       name: Windows x64 Build
+       cache:
+         directories:
+           - $HOME/AppData/Local/Temp/chocolatey
+           - /C/tools/msys64
+       script:
+         - $mingw64 scripts/travis/build_test.sh
 
     - stage: build_release
       os: linux
@@ -105,6 +115,15 @@ jobs:
             - awscli
       script:
         - scripts/travis/external_build.sh ./scripts/travis/integration_test.sh
+    - # same stage, parallel job
+       os: windows
+       name: Windows x64 Build
+       cache:
+         directories:
+           - $HOME/AppData/Local/Temp/chocolatey
+           - /C/tools/msys64
+       script:
+         - $mingw64 scripts/travis/build_test.sh
 
     - stage: deploy
       name: Ubuntu Deploy
@@ -148,8 +167,37 @@ cache:
     - crypto/lib
 
 before_install:
-  # Disable sometimes-broken sources.list in Travis base images
-  - sudo rm -vf /etc/apt/sources.list.d/*
+- |-
+   case $TRAVIS_OS_NAME in
+   linux)
+     # Disable sometimes-broken sources.list in Travis base images
+     sudo rm -vf /etc/apt/sources.list.d/*
+     ;;
+   windows)
+     [[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64
+     choco uninstall -y mingw
+     choco upgrade --no-progress -y msys2
+     export msys2='cmd //C RefreshEnv.cmd '
+     export msys2+='& set MSYS=winsymlinks:nativestrict '
+     export msys2+='& C:\\tools\\msys64\\msys2_shell.cmd -defterm -no-start'
+     export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
+     export msys2+=" -msys2 -c "\"\$@"\" --"
+     $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain unzip
+     ## Install more MSYS2 packages from https://packages.msys2.org/base here
+     taskkill //IM gpg-agent.exe //F  # https://travis-ci.community/t/4967
+     export PATH=/C/tools/msys64/mingw64/bin:$PATH
+     export MAKE=mingw32-make  # so that Autotools can find it
+     ;;
+   esac
+
+before_cache:
+- |-
+   case $TRAVIS_OS_NAME in
+   windows)
+     # https://unix.stackexchange.com/a/137322/107554
+     $msys2 pacman --sync --clean --noconfirm
+     ;;
+   esac
 
 addons:
   apt:

Makefile

@@ -1,16 +1,23 @@
-export GOPATH		:= $(shell go env GOPATH)
-GOPATH1		:= $(firstword $(subst :, ,$(GOPATH)))
+UNAME := $(shell uname)
+ifneq (, $(findstring MINGW,$(UNAME)))
+#Gopath is not saved across sessions, probably existing Windows env vars, override them
+export GOPATH := ${HOME}/go
+GOPATH1 := $(GOPATH)
+export PATH := $(PATH):$(GOPATH)/bin
+else
+export GOPATH := $(shell go env GOPATH)
+GOPATH1 := $(firstword $(subst :, ,$(GOPATH)))
+endif
 export GO111MODULE	:= on
 export GOPROXY := direct
-
-UNAME		:= $(shell uname)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
 S3_RELEASE_BUCKET = $$S3_RELEASE_BUCKET
 
 # If build number already set, use it - to ensure same build number across multiple platforms being built
 BUILDNUMBER      ?= $(shell ./scripts/compute_build_number.sh)
+FULLBUILDNUMBER  ?= $(shell ./scripts/compute_build_number.sh -f)
 COMMITHASH       := $(shell ./scripts/compute_build_commit.sh)
 BUILDBRANCH      := $(shell ./scripts/compute_branch.sh)
 CHANNEL          ?= $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
@@ -38,6 +45,11 @@ endif
 endif
 endif
 
+ifneq (, $(findstring MINGW,$(UNAME)))
+EXTLDFLAGS := -static-libstdc++ -static-libgcc
+export GOBUILDMODE := -buildmode=exe
+endif
+
 GOTAGS      := --tags "$(GOTAGSLIST)"
 GOTRIMPATH	:= $(shell go help build | grep -q .-trimpath && echo -trimpath)
 
@@ -53,7 +65,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger
+MSGP_GENERATE	:= ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger
 
 default: build
 
@@ -289,10 +301,6 @@ install: build
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule
 
-SUPPORTED_ARCHIVE_OS_ARCH = linux/amd64 linux/arm64 linux/arm darwin/amd64
-
 archive:
 	CHANNEL=$(CHANNEL) \
-	PATH=$(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/bin:$${PATH} \
-	scripts/upload_version.sh $(CHANNEL) $(SRCPATH)/tmp/node_pkgs $(S3_RELEASE_BUCKET)
-
+	aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/${CHANNEL}/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*${CHANNEL}*$(FULLBUILDNUMBER)*"

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.com/algorand/go-algorand.svg?token=25XP72ADqbCQJ3TJVC9S&branch=master)](https://travis-ci.com/algorand/go-algorand)
+[![Build Status](https://travis-ci.com/algorand/go-algorand.svg?branch=master)](https://travis-ci.com/algorand/go-algorand)
 
 go-algorand
 ====================

agreement/abstractions.go

@@ -307,3 +307,18 @@ type Message struct {
 type EventsProcessingMonitor interface {
 	UpdateEventsQueue(queueName string, queueLength int)
 }
+
+// LedgerDroppedRoundError is a wrapper error for when the ledger cannot return a Lookup query because
+// the entry is old and was dropped from the ledger. The purpose of this wrapper is to help the
+// agreement differentiate between a malicious vote and a vote that it cannot verify
+type LedgerDroppedRoundError struct {
+	Err error
+}
+
+func (e *LedgerDroppedRoundError) Error() string {
+	return e.Err.Error()
+}
+
+func (e *LedgerDroppedRoundError) Unwrap() error {
+	return e.Err
+}

agreement/agreementtest/simulate.go

@@ -31,6 +31,7 @@ import (
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/logging"
+	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util/db"
 	"github.com/algorand/go-algorand/util/timers"
 )
@@ -68,7 +69,7 @@ func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time {
 		return ta
 	}
 
-	if d == agreement.FilterTimeout() && !i.HasPending("pseudonode") {
+	if d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") {
 		close(ta)
 	}
 	return ta

agreement/asyncVoteVerifier.go

@@ -18,6 +18,7 @@ package agreement
 
 import (
 	"context"
+	"errors"
 	"sync"
 
 	"github.com/algorand/go-algorand/util/execpool"
@@ -104,7 +105,11 @@ func (avv *AsyncVoteVerifier) executeVoteVerification(task interface{}) interfac
 		// request was not cancelled, so we verify it here and return the result on the channel
 		v, err := req.uv.verify(req.l)
 		req.message.Vote = v
-		return &asyncVerifyVoteResponse{v: v, index: req.index, message: req.message, err: err, req: &req}
+
+		var e *LedgerDroppedRoundError
+		cancelled := errors.As(err, &e)
+
+		return &asyncVerifyVoteResponse{v: v, index: req.index, message: req.message, err: err, cancelled: cancelled, req: &req}
 	}
 }
 
@@ -118,7 +123,11 @@ func (avv *AsyncVoteVerifier) executeEqVoteVerification(task interface{}) interf
 	default:
 		// request was not cancelled, so we verify it here and return the result on the channel
 		ev, err := req.uev.verify(req.l)
-		return &asyncVerifyVoteResponse{ev: ev, index: req.index, message: req.message, err: err, req: &req}
+
+		var e *LedgerDroppedRoundError
+		cancelled := errors.As(err, &e)
+
+		return &asyncVerifyVoteResponse{ev: ev, index: req.index, message: req.message, err: err, cancelled: cancelled, req: &req}
 	}
 }
 

agreement/common_test.go

@@ -192,10 +192,14 @@ type testLedger struct {
 	certs     map[basics.Round]Certificate
 	nextRound basics.Round
 
+	maxNumBlocks uint64
+
 	// constant
 	state map[basics.Address]basics.AccountData
 
 	notifications map[basics.Round]signal
+
+	consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)
 }
 
 func makeTestLedger(state map[basics.Address]basics.AccountData) Ledger {
@@ -204,13 +208,54 @@ func makeTestLedger(state map[basics.Address]basics.AccountData) Ledger {
 	l.certs = make(map[basics.Round]Certificate)
 	l.nextRound = 1
 
-	// deep copy of state
 	l.state = make(map[basics.Address]basics.AccountData)
 	for k, v := range state {
 		l.state[k] = v
 	}
 
 	l.notifications = make(map[basics.Round]signal)
+
+	l.consensusVersion = func(r basics.Round) (protocol.ConsensusVersion, error) {
+		return protocol.ConsensusCurrentVersion, nil
+	}
+	return l
+}
+
+func makeTestLedgerWithConsensusVersion(state map[basics.Address]basics.AccountData, consensusVersion func(basics.Round) (protocol.ConsensusVersion, error)) Ledger {
+	l := new(testLedger)
+	l.entries = make(map[basics.Round]bookkeeping.Block)
+	l.certs = make(map[basics.Round]Certificate)
+	l.nextRound = 1
+
+	l.state = make(map[basics.Address]basics.AccountData)
+	for k, v := range state {
+		l.state[k] = v
+	}
+
+	l.notifications = make(map[basics.Round]signal)
+
+	l.consensusVersion = consensusVersion
+	return l
+}
+
+func makeTestLedgerMaxBlocks(state map[basics.Address]basics.AccountData, maxNumBlocks uint64) Ledger {
+	l := new(testLedger)
+	l.entries = make(map[basics.Round]bookkeeping.Block)
+	l.certs = make(map[basics.Round]Certificate)
+	l.nextRound = 1
+
+	l.maxNumBlocks = maxNumBlocks
+
+	l.state = make(map[basics.Address]basics.AccountData)
+	for k, v := range state {
+		l.state[k] = v
+	}
+
+	l.notifications = make(map[basics.Round]signal)
+
+	l.consensusVersion = func(r basics.Round) (protocol.ConsensusVersion, error) {
+		return protocol.ConsensusCurrentVersion, nil
+	}
 	return l
 }
 
@@ -268,6 +313,10 @@ func (l *testLedger) LookupDigest(r basics.Round) (crypto.Digest, error) {
 		panic(err)
 	}
 
+	if l.maxNumBlocks != 0 && r+round(l.maxNumBlocks) < l.nextRound {
+		return crypto.Digest{}, &LedgerDroppedRoundError{}
+	}
+
 	return l.entries[r].Digest(), nil
 }
 
@@ -279,6 +328,11 @@ func (l *testLedger) Lookup(r basics.Round, a basics.Address) (basics.AccountDat
 		err := fmt.Errorf("Lookup called on future round: %v >= %v! (this is probably a bug)", r, l.nextRound)
 		panic(err)
 	}
+
+	if l.maxNumBlocks != 0 && r+round(l.maxNumBlocks) < l.nextRound {
+		return basics.AccountData{}, &LedgerDroppedRoundError{}
+	}
+
 	return l.state[a], nil
 }
 
@@ -348,11 +402,15 @@ func (l *testLedger) EnsureDigest(c Certificate, verifier *AsyncVoteVerifier) {
 }
 
 func (l *testLedger) ConsensusParams(r basics.Round) (config.ConsensusParams, error) {
-	return config.Consensus[protocol.ConsensusCurrentVersion], nil
+	version, err := l.ConsensusVersion(r)
+	if err != nil {
+		return config.ConsensusParams{}, err
+	}
+	return config.Consensus[version], nil
 }
 
 func (l *testLedger) ConsensusVersion(r basics.Round) (protocol.ConsensusVersion, error) {
-	return protocol.ConsensusCurrentVersion, nil
+	return l.consensusVersion(r)
 }
 
 // simulation helpers