Merge pull request #1954 from onetechnical/onetechnical/relbeta2.5.1

go-algorand 2.5.1-beta

Author: algojohnlee

.github/ISSUE_TEMPLATE/algorand-engineering-team-issue-template.md

@@ -9,10 +9,10 @@ assignees: ''
 ---
 
 ## Summary
-*Describe the problem identified or the general goal of this issue*
+<!-- *Describe the problem identified or the general goal of this issue* -->
 
 ## Scope/Requirements
-*What's involved in this issue? What's required to achieve the goal?*
+<!-- *What's involved in this issue? What's required to achieve the goal?* -->
 
 ## Urgency/Relative Priority
-*How urgent is this issue? What are the timing considerations to take into account?*
+<!-- *How urgent is this issue? What are the timing considerations to take into account?* -->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,27 +7,24 @@ assignees: ''
 
 ---
 
-<!--
-NOTE: If this issue relates to security, please use the vulnerability disclosure form here:
-https://www.algorand.com/resources/blog/security
-
-General, developer or support questions concerning Algorand should be directed to the Algorand Forums https://forum.algorand.org/.
--->
-
 ### Subject of the issue
-Describe your issue here.
+
+<!-- Describe your issue here. -->
 
 ### Your environment
+
+<!--
 * Software version: `algod -v`
 * Node status if applicable: `goal node status`
 * Operating System details. 
 * In many cases log files and cadaver files are also useful to include. Since these files may be large, an Algorand developer may request them later. These files may include public addresses that you're participating with. If that is a concern please be sure to scrub that data.
+-->
 
 ### Steps to reproduce
-Tell us how to reproduce this issue.
+
+1.
+2.
 
 ### Expected behaviour
-Tell us what should happen
 
 ### Actual behaviour
-Tell us what happens instead

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,4 @@
+contact_links:
+  - name: ❓ Question and Help
+    url: https://forum.algorand.org/
+    about: The issue tracker is not for support questions. Please join our community for help and discussion.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,27 +7,14 @@ assignees: ''
 
 ---
 
-<!--
-NOTE: If this issue relates to security, please use the vulnerability disclosure form here:
-https://www.algorand.com/resources/blog/security
+## Is your feature request related to a problem? Please describe.
 
-General, developer or support questions concerning Algorand should be directed to the Algorand Forums https://forum.algorand.org/.
--->
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
 
-<!--
-This project is focused on low level platform concerns. A good feature request would be related
-to protocol and network usage, debugging and monitoring tools, and goal commands.
+## Describe the solution you'd like.
 
-Higher level suggestions relating to layer 2 applications (like wallet apps) are not appropriate here.
--->
-**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what you want to happen. -->
 
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+## Additional context.
 
-**Describe the solution you'd like**
-
-A clear and concise description of what you want to happen.
-
-**Additional context**
-
-Add any other context or screenshots about the feature request here.
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/question.md

@@ -15,8 +15,8 @@ It is also a good idea to run tests:
 
 ## Summary
 
-Explain the goal of this change and what problem it is solving. Format this cleanly so that it may be used for a commit message, as your changes will be squash-merged.
+<!-- Explain the goal of this change and what problem it is solving. Format this cleanly so that it may be used for a commit message, as your changes will be squash-merged. -->
 
 ## Test Plan
 
-How did you test these changes? Please provide the exact scenarios you tested in as much detail as possible including commands, output and rationale.
+<!-- How did you test these changes? Please provide the exact scenarios you tested in as much detail as possible including commands, output and rationale. -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,7 @@
+# Vulnerability Disclosures
+
+Algorand takes the security of the platform and of its users very seriously. We recognize the important role of external security researchers and developers in helping keep our community safe. As with most security reward programs, we ask that you use common sense when looking for security bugs. Vulnerabilities must be disclosed to us privately with reasonable time to respond, and avoid compromise of other users and accounts, or loss of funds that are not your own. We do not reward denial of service, spam, or social engineering vulnerabilities.
+
+If you believe that you have found a security vulnerability you may disclose it here:
+
+https://www.algorand.com/resources/blog/security

SECURITY.md

@@ -12,6 +12,7 @@ In no particular order:
 - jsign
 - RomitKumar
 - jeapostrophe
+- aybehrouz
 
 ### Bug Reports
 - Nanyan

THANKS.md

@@ -55,7 +55,7 @@ func seedRound(r basics.Round, cparams config.ConsensusParams) basics.Round {
 	return r.SubSaturate(basics.Round(cparams.SeedLookback))
 }
 
-// a helper function for obtaining memberhship verification parameters.
+// a helper function for obtaining membership verification parameters.
 func membership(l LedgerReader, addr basics.Address, r basics.Round, p period, s step) (m committee.Membership, err error) {
 	cparams, err := l.ConsensusParams(ParamsRound(r))
 	if err != nil {

agreement/selector.go

@@ -1 +1 @@
-0
+1

buildnumber.dat

@@ -407,7 +407,7 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro
 			}
 			return cs.abort(fmt.Errorf("processStageLastestBlockDownload failed when calling VerifyCatchpoint : %v", err))
 		}
-		// give a rank to the download, as the download was successfull.
+		// give a rank to the download, as the download was successful.
 		peerRank := cs.blocksDownloadPeerSelector.PeerDownloadDurationToRank(peer, blockDownloadDuration)
 		cs.blocksDownloadPeerSelector.RankPeer(peer, peerRank)
 

catchup/catchpointService.go

@@ -54,14 +54,13 @@ type FetcherFactory interface {
 	// Create a new fetcher
 	New() Fetcher
 	// Create a new fetcher that also fetches from backup peers over gossip network utilising given message tag
-	NewOverGossip(requestTag protocol.Tag) Fetcher
+	NewOverGossip() Fetcher
 }
 
 // NetworkFetcherFactory creates network fetchers
 type NetworkFetcherFactory struct {
 	net       network.GossipNode
 	peerLimit int
-	fs        *rpcs.WsFetcherService
 	cfg       *config.Local
 
 	log logging.Logger
@@ -78,12 +77,11 @@ func (factory NetworkFetcherFactory) makeHTTPFetcherFromPeer(log logging.Logger,
 
 // MakeNetworkFetcherFactory returns a network fetcher factory, that associates fetchers with no more than peerLimit peers from the aggregator.
 // WSClientSource can be nil, if no network exists to create clients from (defaults to http clients)
-func MakeNetworkFetcherFactory(net network.GossipNode, peerLimit int, fs *rpcs.WsFetcherService, cfg *config.Local) NetworkFetcherFactory {
+func MakeNetworkFetcherFactory(net network.GossipNode, peerLimit int, cfg *config.Local) NetworkFetcherFactory {
 	var factory NetworkFetcherFactory
 	factory.net = net
 	factory.peerLimit = peerLimit
 	factory.log = logging.Base()
-	factory.fs = fs
 	factory.cfg = cfg
 	return factory
 }
@@ -119,19 +117,14 @@ func (factory NetworkFetcherFactory) New() Fetcher {
 // NewOverGossip returns a fetcher using the given message tag.
 // If there are gossip peers, then it returns a fetcher over gossip
 // Otherwise, it returns an HTTP fetcher
-// We should never build two fetchers utilising the same tag. Why?
-func (factory NetworkFetcherFactory) NewOverGossip(tag protocol.Tag) Fetcher {
+func (factory NetworkFetcherFactory) NewOverGossip() Fetcher {
 	gossipPeers := factory.net.GetPeers(network.PeersConnectedIn)
 	factory.log.Debugf("%d gossip peers", len(gossipPeers))
 	if len(gossipPeers) == 0 {
 		factory.log.Info("no gossip peers for NewOverGossip")
 		return factory.New()
 	}
-	if factory.fs == nil {
-		factory.log.Info("WsFetcherService not available; fetch over gossip disabled")
-		return factory.New()
-	}
-	f := MakeWsFetcher(factory.log, tag, gossipPeers, factory.fs, factory.cfg)
+	f := MakeWsFetcher(factory.log, gossipPeers, factory.cfg)
 	return &ComposedFetcher{fetchers: []Fetcher{factory.New(), f}}
 }
 

catchup/fetcher.go

@@ -131,7 +131,7 @@ func getAllClientsSelectedForRound(t *testing.T, fetcher *NetworkFetcher, round
 func TestSelectValidRemote(t *testing.T) {
 	network := makeMockClientAggregator(t, false, false)
 	cfg := config.GetDefaultLocal()
-	factory := MakeNetworkFetcherFactory(network, numberOfPeers, nil, &cfg)
+	factory := MakeNetworkFetcherFactory(network, numberOfPeers, &cfg)
 	factory.log = logging.TestingLog(t)
 	fetcher := factory.New()
 	require.Equal(t, numberOfPeers, len(fetcher.(*NetworkFetcher).peers))
@@ -596,7 +596,7 @@ func TestGetBlockHTTP(t *testing.T) {
 	_, ok := net.GetPeers(network.PeersConnectedOut)[0].(network.HTTPPeer)
 	require.True(t, ok)
 	cfg := config.GetDefaultLocal()
-	factory := MakeNetworkFetcherFactory(net, numberOfPeers, nil, &cfg)
+	factory := MakeNetworkFetcherFactory(net, numberOfPeers, &cfg)
 	factory.log = logging.TestingLog(t)
 	fetcher := factory.New()
 	// we have one peer, the HTTP block server
@@ -709,7 +709,7 @@ func TestGetBlockMocked(t *testing.T) {
 	require.NoError(t, ledgerA.AddBlock(b, agreement.Certificate{Round: next}))
 
 	// B tries to fetch block
-	factory := MakeNetworkFetcherFactory(nodeB, 10, nil, &cfg)
+	factory := MakeNetworkFetcherFactory(nodeB, 10, &cfg)
 	factory.log = logging.TestingLog(t)
 	nodeBRPC := factory.New()
 	ctx, cf := context.WithTimeout(context.Background(), time.Second)
@@ -759,7 +759,7 @@ func TestGetFutureBlock(t *testing.T) {
 	rpcs.MakeBlockService(config.GetDefaultLocal(), ledgerA, nodeA, "test genesisID")
 
 	// B tries to fetch block 4
-	factory := MakeNetworkFetcherFactory(nodeB, 10, nil, &cfg)
+	factory := MakeNetworkFetcherFactory(nodeB, 10, &cfg)
 	factory.log = logging.TestingLog(t)
 	nodeBRPC := factory.New()
 	ctx, cf := context.WithTimeout(context.Background(), time.Second)
@@ -851,7 +851,7 @@ func makeTestUnicastPeer(gn network.GossipNode, version string, t *testing.T) ne
 // A quick GetBlock over websockets test hitting a mocked websocket server (no actual connection)
 func TestGetBlockWS(t *testing.T) {
 	// test the WS fetcher:
-	// 1. fetcher sends UniCatchupReqTag to http peer
+	// 1. fetcher sends UniEnsBlockReqTag to http peer
 	// 2. peer send message to gossip node
 	// 3. gossip node send message to ledger service
 	// 4. ledger service responds with UniCatchupResTag sending it back to the http peer
@@ -867,7 +867,7 @@ func TestGetBlockWS(t *testing.T) {
 
 	cfg := config.GetDefaultLocal()
 
-	versions := []string{"1", "2.1"}
+	versions := []string{"2.1"}
 	for _, version := range versions { // range network.SupportedProtocolVersions {
 
 		net := &httpTestPeerSource{}
@@ -881,27 +881,21 @@ func TestGetBlockWS(t *testing.T) {
 		up := makeTestUnicastPeer(net, version, t)
 		net.peers = append(net.peers, up)
 
-		fs := rpcs.MakeWsFetcherService(logging.TestingLog(t), net)
-		fs.Start()
-
 		_, ok := net.GetPeers(network.PeersConnectedIn)[0].(network.UnicastPeer)
 		require.True(t, ok)
-		factory := MakeNetworkFetcherFactory(net, numberOfPeers, fs, &cfg)
+		factory := MakeNetworkFetcherFactory(net, numberOfPeers, &cfg)
 		factory.log = logging.TestingLog(t)
-		fetcher := factory.NewOverGossip(protocol.UniCatchupReqTag)
+		fetcher := factory.NewOverGossip()
 		// we have one peer, the Ws block server
 		require.Equal(t, fetcher.NumPeers(), 1)
 
 		var block *bookkeeping.Block
 		var cert *agreement.Certificate
 		var client FetcherClient
 
-		//		start := time.Now()
 		block, cert, client, err = fetcher.FetchBlock(context.Background(), next)
 		require.NotNil(t, client)
 		require.NoError(t, err)
-		//		end := time.Now()
-		//		require.True(t, end.Sub(start) < 10*time.Second)
 		require.Equal(t, &b, block)
 		if err == nil {
 			require.NotEqual(t, nil, block)

catchup/fetcher_test.go

@@ -56,7 +56,7 @@ func BenchmarkServiceFetchBlocks(b *testing.B) {
 		require.NoError(b, err)
 
 		// Make Service
-		syncer := MakeService(logging.Base(), defaultConfig, net, local, nil, new(mockedAuthenticator), nil)
+		syncer := MakeService(logging.Base(), defaultConfig, net, local, new(mockedAuthenticator), nil)
 		syncer.fetcherFactory = makeMockFactory(&MockedFetcher{ledger: remote, timeout: false, tries: make(map[basics.Round]int), latency: 100 * time.Millisecond, predictable: true})
 
 		b.StartTimer()

catchup/pref_test.go

@@ -33,7 +33,6 @@ import (
 	"github.com/algorand/go-algorand/logging/telemetryspec"
 	"github.com/algorand/go-algorand/network"
 	"github.com/algorand/go-algorand/protocol"
-	"github.com/algorand/go-algorand/rpcs"
 )
 
 const catchupPeersForSync = 10
@@ -103,17 +102,16 @@ type BlockAuthenticator interface {
 }
 
 // MakeService creates a catchup service instance from its constituent components
-// If wsf is nil, then fetch over gossip is disabled.
-func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, wsf *rpcs.WsFetcherService, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate) (s *Service) {
+func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate) (s *Service) {
 	s = &Service{}
 
 	s.cfg = config
-	s.fetcherFactory = MakeNetworkFetcherFactory(net, catchupPeersForSync, wsf, &config)
+	s.fetcherFactory = MakeNetworkFetcherFactory(net, catchupPeersForSync, &config)
 	s.ledger = ledger
 	s.net = net
 	s.auth = auth
 	s.unmatchedPendingCertificates = unmatchedPendingCertificates
-	s.latestRoundFetcherFactory = MakeNetworkFetcherFactory(net, blockQueryPeerLimit, wsf, &config)
+	s.latestRoundFetcherFactory = MakeNetworkFetcherFactory(net, blockQueryPeerLimit, &config)
 	s.log = log.With("Context", "sync")
 	s.parallelBlocks = config.CatchupParallelBlocks
 	s.deadlineTimeout = agreement.DeadlineTimeout()
@@ -324,7 +322,7 @@ func (s *Service) pipelineCallback(fetcher Fetcher, r basics.Round, thisFetchCom
 
 // TODO the following code does not handle the following case: seedLookback upgrades during fetch
 func (s *Service) pipelinedFetch(seedLookback uint64) {
-	fetcher := s.fetcherFactory.NewOverGossip(protocol.UniCatchupReqTag)
+	fetcher := s.fetcherFactory.NewOverGossip()
 	defer fetcher.Close()
 
 	// make sure that we have at least one peer
@@ -559,7 +557,7 @@ func (s *Service) syncCert(cert *PendingUnmatchedCertificate) {
 // TODO this doesn't actually use the digest from cert!
 func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.AsyncVoteVerifier) {
 	blockHash := bookkeeping.BlockHash(cert.Proposal.BlockDigest) // semantic digest (i.e., hash of the block header), not byte-for-byte digest
-	fetcher := s.latestRoundFetcherFactory.NewOverGossip(protocol.UniEnsBlockReqTag)
+	fetcher := s.latestRoundFetcherFactory.NewOverGossip()
 	defer func() {
 		fetcher.Close()
 	}()
@@ -569,7 +567,7 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy
 			// refresh peers and try again
 			logging.Base().Warn("fetchRound found no outgoing peers")
 			s.net.RequestConnectOutgoing(true, s.ctx.Done())
-			fetcher = s.latestRoundFetcherFactory.NewOverGossip(protocol.UniEnsBlockReqTag)
+			fetcher = s.latestRoundFetcherFactory.NewOverGossip()
 		}
 		// Ask the fetcher to get the block somehow
 		block, fetchedCert, rpcc, err := s.innerFetch(fetcher, cert.Round)