Merge pull request #1748 from onetechnical/onetechnical/relstable2.3.0

go-algorand 2.3.0-stable

Author: algojohnlee

scripts/release/mule/deploy/deb/.aptly.conf renamed to .aptly.conf

@@ -1,7 +1,7 @@
 UNAME := $(shell uname)
-ifneq (, $(findstring MINGW,$(UNAME)))
+ifneq (,$(findstring MINGW,$(UNAME)))
 #Gopath is not saved across sessions, probably existing Windows env vars, override them
-export GOPATH := ${HOME}/go
+export GOPATH := $(HOME)/go
 GOPATH1 := $(GOPATH)
 export PATH := $(PATH):$(GOPATH)/bin
 else
@@ -26,6 +26,9 @@ DEFAULT_DEADLOCK ?= $(shell ./scripts/compute_branch_deadlock_default.sh $(BUILD
 
 GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 
+# e.g. make GOTAGSCUSTOM=msgtrace
+GOTAGSLIST += ${GOTAGSCUSTOM}
+
 ifeq ($(UNAME), Linux)
 EXTLDFLAGS := -static-libstdc++ -static-libgcc
 ifeq ($(ARCH), amd64)
@@ -46,12 +49,12 @@ endif
 endif
 
 ifneq (, $(findstring MINGW,$(UNAME)))
-EXTLDFLAGS := -static-libstdc++ -static-libgcc
+EXTLDFLAGS := -static -static-libstdc++ -static-libgcc
 export GOBUILDMODE := -buildmode=exe
 endif
 
 GOTAGS      := --tags "$(GOTAGSLIST)"
-GOTRIMPATH	:= $(shell go help build | grep -q .-trimpath && echo -trimpath)
+GOTRIMPATH	:= $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath)
 
 GOLDFLAGS_BASE  := -X github.com/algorand/go-algorand/config.BuildNumber=$(BUILDNUMBER) \
 		 -X github.com/algorand/go-algorand/config.CommitHash=$(COMMITHASH) \
@@ -62,8 +65,8 @@ GOLDFLAGS_BASE  := -X github.com/algorand/go-algorand/config.BuildNumber=$(BUILD
 GOLDFLAGS := $(GOLDFLAGS_BASE) \
 		 -X github.com/algorand/go-algorand/config.Channel=$(CHANNEL)
 
-UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ ))
-ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... ))
+UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
+ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
 MSGP_GENERATE	:= ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger
 
@@ -302,5 +305,4 @@ install: build
 include ./scripts/release/mule/Makefile.mule
 
 archive:
-	CHANNEL=$(CHANNEL) \
-	aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/${CHANNEL}/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*${CHANNEL}*$(FULLBUILDNUMBER)*"
+	aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/$(CHANNEL)/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*$(FULLBUILDNUMBER)*"

Makefile

@@ -25,6 +25,7 @@ import (
 	"github.com/algorand/go-algorand/agreement"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/network"
+	"github.com/algorand/go-algorand/network/messagetracer"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util/metrics"
 )
@@ -50,6 +51,8 @@ type networkImpl struct {
 
 	net network.GossipNode
 	log logging.Logger
+
+	trace messagetracer.MessageTracer
 }
 
 // WrapNetwork adapts a network.GossipNode into an agreement.Network.
@@ -66,6 +69,12 @@ func WrapNetwork(net network.GossipNode, log logging.Logger) agreement.Network {
 	return i
 }
 
+// SetTrace modifies the result of WrapNetwork to add network propagation tracing
+func SetTrace(net agreement.Network, trace messagetracer.MessageTracer) {
+	i := net.(*networkImpl)
+	i.trace = trace
+}
+
 func (i *networkImpl) Start() {
 	handlers := []network.TaggedMessageHandler{
 		{Tag: protocol.AgreementVoteTag, MessageHandler: network.HandlerFunc(i.processVoteMessage)},
@@ -87,6 +96,9 @@ func (i *networkImpl) processVoteMessage(raw network.IncomingMessage) network.Ou
 }
 
 func (i *networkImpl) processProposalMessage(raw network.IncomingMessage) network.OutgoingMessage {
+	if i.trace != nil {
+		i.trace.HashTrace(messagetracer.Proposal, raw.Data)
+	}
 	return i.processMessage(raw, i.proposalCh)
 }
 

agreement/gossip/network.go

@@ -43,6 +43,7 @@ type CatchpointCatchupStats struct {
 	CatchpointLabel   string
 	TotalAccounts     uint64
 	ProcessedAccounts uint64
+	VerifiedAccounts  uint64
 	TotalBlocks       uint64
 	AcquiredBlocks    uint64
 	VerifiedBlocks    uint64
@@ -262,7 +263,7 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 	}
 
 	// download balances file.
-	ledgerFetcher := makeLedgerFetcher(cs.net, cs.ledgerAccessor, cs.log, cs)
+	ledgerFetcher := makeLedgerFetcher(cs.net, cs.ledgerAccessor, cs.log, cs, cs.config)
 	attemptsCount := 0
 
 	for {
@@ -277,11 +278,15 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 		}
 		err = ledgerFetcher.downloadLedger(cs.ctx, round)
 		if err == nil {
-			break
+			err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedAccounts)
+			if err == nil {
+				break
+			}
+			// failed to build the merkle trie for the above catchpoint file.
 		}
 		// instead of testing for err == cs.ctx.Err() , we'll check on the context itself.
 		// this is more robust, as the http client library sometimes wrap the context canceled
-		// error with other erros.
+		// error with other errors.
 		if cs.ctx.Err() != nil {
 			return cs.stopOrAbort()
 		}
@@ -300,6 +305,13 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 	return nil
 }
 
+// updateVerifiedAccounts update the user's statistics for the given verified accounts
+func (cs *CatchpointCatchupService) updateVerifiedAccounts(verifiedAccounts uint64) {
+	cs.statsMu.Lock()
+	defer cs.statsMu.Unlock()
+	cs.stats.VerifiedAccounts = verifiedAccounts
+}
+
 // processStageLastestBlockDownload is the third catchpoint catchup stage. It downloads the latest block and verify that against the previously downloaded ledger.
 func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err error) {
 	blockRound, err := cs.ledgerAccessor.GetCatchupBlockRound(cs.ctx)
@@ -328,6 +340,7 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro
 				if attemptsCount <= cs.config.CatchupBlockDownloadRetryAttempts {
 					// try again.
 					blk = nil
+					cs.log.Infof("processStageLastestBlockDownload: block %d download failed, another attempt will be made; err = %v", blockRound, err)
 					continue
 				}
 				return cs.abort(fmt.Errorf("processStageLastestBlockDownload failed to get block %d : %v", blockRound, err))
@@ -369,6 +382,7 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro
 			if attemptsCount <= cs.config.CatchupBlockDownloadRetryAttempts {
 				// try again.
 				blk = nil
+				cs.log.Infof("processStageLastestBlockDownload: block %d verification against catchpoint failed, another attempt will be made; err = %v", blockRound, err)
 				continue
 			}
 			return cs.abort(fmt.Errorf("processStageLastestBlockDownload failed when calling VerifyCatchpoint : %v", err))

catchup/catchpointService.go

@@ -27,6 +27,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/logging"
@@ -39,12 +40,8 @@ var errNoLedgerForRound = errors.New("No ledger available for given round")
 const (
 	// maxCatchpointFileChunkSize is a rough estimate for the worst-case scenario we're going to have of all the accounts data per a single catchpoint file chunk.
 	maxCatchpointFileChunkSize = ledger.BalancesPerCatchpointFileChunk * basics.MaxEncodedAccountDataSize
-	// maxCatchpointFileDownloadDuration is the maximum time we would wait for download an entire catchpoint file
-	maxCatchpointFileDownloadDuration = 45 * time.Minute
-	// expectedWorstDownloadSpeedBytesPerSecond defines the worst case-scenario download speed we expect to get while downloading a catchpoint file
-	expectedWorstDownloadSpeedBytesPerSecond = 20 * 1024
-	// maxCatchpointFileChunkDownloadDuration is the maximum amount of time we would wait to download a single chunk off a catchpoint file
-	maxCatchpointFileChunkDownloadDuration = 2*time.Minute + maxCatchpointFileChunkSize*time.Second/expectedWorstDownloadSpeedBytesPerSecond
+	// defaultMinCatchpointFileDownloadBytesPerSecond defines the worst-case scenario download speed we expect to get while downloading a catchpoint file
+	defaultMinCatchpointFileDownloadBytesPerSecond = 20 * 1024
 	// catchpointFileStreamReadSize defines the number of bytes we would attempt to read at each itration from the incoming http data stream
 	catchpointFileStreamReadSize = 4096
 )
@@ -62,14 +59,16 @@ type ledgerFetcher struct {
 	log      logging.Logger
 	peers    []network.Peer
 	reporter ledgerFetcherReporter
+	config   config.Local
 }
 
-func makeLedgerFetcher(net network.GossipNode, accessor ledger.CatchpointCatchupAccessor, log logging.Logger, reporter ledgerFetcherReporter) *ledgerFetcher {
+func makeLedgerFetcher(net network.GossipNode, accessor ledger.CatchpointCatchupAccessor, log logging.Logger, reporter ledgerFetcherReporter, cfg config.Local) *ledgerFetcher {
 	return &ledgerFetcher{
 		net:      net,
 		accessor: accessor,
 		log:      log,
 		reporter: reporter,
+		config:   cfg,
 	}
 }
 
@@ -101,7 +100,8 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee
 	if err != nil {
 		return err
 	}
-	timeoutContext, timeoutContextCancel := context.WithTimeout(ctx, maxCatchpointFileDownloadDuration)
+
+	timeoutContext, timeoutContextCancel := context.WithTimeout(ctx, lf.config.MaxCatchpointDownloadDuration)
 	defer timeoutContextCancel()
 	request = request.WithContext(timeoutContext)
 	network.SetUserAgentHeader(request.Header)
@@ -133,6 +133,15 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee
 		err = fmt.Errorf("getPeerLedger : http ledger fetcher response has an invalid content type : %s", contentTypes[0])
 		return err
 	}
+
+	// maxCatchpointFileChunkDownloadDuration is the maximum amount of time we would wait to download a single chunk off a catchpoint file
+	maxCatchpointFileChunkDownloadDuration := 2 * time.Minute
+	if lf.config.MinCatchpointFileDownloadBytesPerSecond > 0 {
+		maxCatchpointFileChunkDownloadDuration += maxCatchpointFileChunkSize * time.Second / time.Duration(lf.config.MinCatchpointFileDownloadBytesPerSecond)
+	} else {
+		maxCatchpointFileChunkDownloadDuration += maxCatchpointFileChunkSize * time.Second / defaultMinCatchpointFileDownloadBytesPerSecond
+	}
+
 	watchdogReader := makeWatchdogStreamReader(response.Body, catchpointFileStreamReadSize, 2*maxCatchpointFileChunkSize, maxCatchpointFileChunkDownloadDuration)
 	defer watchdogReader.Close()
 	tarReader := tar.NewReader(watchdogReader)

catchup/ledgerFetcher.go

@@ -26,6 +26,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/go-algorand/components/mocks"
+	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/logging"
@@ -38,7 +39,7 @@ func (lf *dummyLedgerFetcherReporter) updateLedgerFetcherProgress(*ledger.Catchp
 }
 
 func TestNoPeersAvailable(t *testing.T) {
-	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{})
+	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal())
 	err := lf.downloadLedger(context.Background(), basics.Round(0))
 	require.Equal(t, errNoPeersAvailable, err)
 	lf.peers = append(lf.peers, &lf) // The peer is an opaque interface.. we can add anything as a Peer.
@@ -47,7 +48,7 @@ func TestNoPeersAvailable(t *testing.T) {
 }
 
 func TestNonParsableAddress(t *testing.T) {
-	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{})
+	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal())
 	peer := testHTTPPeer(":def")
 	err := lf.getPeerLedger(context.Background(), &peer, basics.Round(0))
 	require.Error(t, err)
@@ -75,7 +76,7 @@ func TestLedgerFetcherErrorResponseHandling(t *testing.T) {
 		w.WriteHeader(httpServerResponse)
 	})
 
-	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{})
+	lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal())
 	peer := testHTTPPeer(listener.Addr().String())
 	err = lf.getPeerLedger(context.Background(), &peer, basics.Round(0))
 	require.Equal(t, errNoLedgerForRound, err)

catchup/ledgerFetcher_test.go

@@ -570,10 +570,12 @@ func printAccountInfo(client libgoal.Client, address string, account v1.Account)
 		if len(name) == 0 {
 			name = "<unnamed>"
 		}
+		_, name = unicodePrintable(name)
 		units := assetParams.UnitName
 		if len(units) == 0 {
 			units = "units"
 		}
+		_, units = unicodePrintable(units)
 		total := assetDecimalsFmt(assetParams.Total, assetParams.Decimals)
 		url := ""
 		if len(assetParams.URL) != 0 {
@@ -602,11 +604,13 @@ func printAccountInfo(client libgoal.Client, address string, account v1.Account)
 		if len(assetName) == 0 {
 			assetName = "<unnamed>"
 		}
+		_, assetName = unicodePrintable(assetName)
 
 		unitName := assetParams.UnitName
 		if len(unitName) == 0 {
 			unitName = "units"
 		}
+		_, unitName = unicodePrintable(unitName)
 
 		frozen := ""
 		if assetHolding.Frozen {

cmd/goal/account.go

@@ -228,7 +228,8 @@ func (accountList *AccountsList) outputAccount(addr string, acctInfo v1.Account,
 	if len(acctInfo.AssetParams) > 0 {
 		var out []string
 		for curid, params := range acctInfo.AssetParams {
-			out = append(out, fmt.Sprintf("%d (%d %s)", curid, params.Total, params.UnitName))
+			_, unitName := unicodePrintable(params.UnitName)
+			out = append(out, fmt.Sprintf("%d (%d %s)", curid, params.Total, unitName))
 		}
 		fmt.Printf("\t[created asset IDs: %s]", strings.Join(out, ", "))
 	}

cmd/goal/accountsList.go

@@ -67,7 +67,7 @@ func init() {
 	createAssetCmd.MarkFlagRequired("creator")
 
 	destroyAssetCmd.Flags().StringVar(&assetManager, "manager", "", "Manager account to issue the destroy transaction (defaults to creator)")
-	destroyAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address for asset to destroy")
+	destroyAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Creator account address for asset to destroy")
 	destroyAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "Asset ID to destroy")
 	destroyAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "Unit name of asset to destroy")
 
@@ -256,7 +256,7 @@ var createAssetCmd = &cobra.Command{
 var destroyAssetCmd = &cobra.Command{
 	Use:   "destroy",
 	Short: "Destroy an asset",
-	Long:  `Issue a transaction deleting an asset from the network. This transaction must be issued by the asset owner, who must hold all outstanding asset tokens.`,
+	Long:  `Issue a transaction deleting an asset from the network. This transaction must be issued by the asset manager while the creator holds all of the asset's tokens.`,
 	Args:  validateNoPosArgsFn,
 	Run: func(cmd *cobra.Command, _ []string) {
 		checkTxValidityPeriodCmdFlags(cmd)
@@ -603,8 +603,8 @@ var infoAssetCmd = &cobra.Command{
 
 		fmt.Printf("Asset ID:         %d\n", assetID)
 		fmt.Printf("Creator:          %s\n", params.Creator)
-		fmt.Printf("Asset name:       %s\n", params.AssetName)
-		fmt.Printf("Unit name:        %s\n", params.UnitName)
+		reportInfof("Asset name:       %s\n", params.AssetName)
+		reportInfof("Unit name:        %s\n", params.UnitName)
 		fmt.Printf("Maximum issue:    %s %s\n", assetDecimalsFmt(params.Total, params.Decimals), params.UnitName)
 		fmt.Printf("Reserve amount:   %s %s\n", assetDecimalsFmt(res.Amount, params.Decimals), params.UnitName)
 		fmt.Printf("Issued:           %s %s\n", assetDecimalsFmt(params.Total-res.Amount, params.Decimals), params.UnitName)

cmd/goal/asset.go

@@ -64,12 +64,12 @@ const (
 	infoNodeStatus                    = "Last committed block: %d\nTime since last block: %s\nSync Time: %s\nLast consensus protocol: %s\nNext consensus protocol: %s\nRound for next consensus protocol: %d\nNext consensus protocol supported: %v"
 	catchupStoppedOnUnsupported       = "Last supported block (%d) is committed. The next block consensus protocol is not supported. Catchup service is stopped."
 	infoNodeCatchpointCatchupStatus   = "Last committed block: %d\nSync Time: %s\nCatchpoint: %s"
-	infoNodeCatchpointCatchupAccounts = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d"
+	infoNodeCatchpointCatchupAccounts = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d\nCatchpoint accounts verified: %d"
 	infoNodeCatchpointCatchupBlocks   = "Catchpoint total blocks: %d\nCatchpoint downloaded blocks: %d"
 	nodeLastCatchpoint                = "Last Catchpoint: %s"
 	errorNodeCreationIPFailure        = "Parsing passed IP %v failed: need a valid IPv4 or IPv6 address with a specified port number"
 	errorNodeNotDetected              = "Algorand node does not appear to be running: %s"
-	errorNodeStatus                   = "Cannot contact Algorand node: %s."
+	errorNodeStatus                   = "Cannot contact Algorand node: %s"
 	errorNodeFailedToStart            = "Algorand node failed to start: %s"
 	errorNodeRunning                  = "Node must be stopped before writing APIToken"
 	errorNodeFailGenToken             = "Cannot generate API token: %s"