Merge pull request #6145 from Algo-devops-service/relstable3.26.0

Author: algojohnlee

.circleci/config.yml

@@ -45,30 +45,14 @@ executors:
     machine:
       image: << pipeline.parameters.ubuntu_image >>
     resource_class: arm.large
-  mac_amd64_medium:
+  mac_arm64_medium:
     macos:
       xcode: 14.2.0
-    resource_class: macos.x86.medium.gen2
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: "true"
-  mac_amd64_large:
+    resource_class: macos.m1.medium.gen1
+  mac_arm64_large:
     macos:
       xcode: 14.2.0
-    # Since they removed the large class for amd64, we will use medium here too.
-    resource_class: macos.x86.medium.gen2
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: "true"
-  mac_arm64: &executor-mac-arm64
-    machine: true
-    resource_class: algorand/macstadium-m1
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: "true"
-  # these are required b/c jobs explicitly assign sizes to the executors
-  # for `mac_arm64` there is only one size
-  mac_arm64_medium:
-    <<: *executor-mac-arm64
-  mac_arm64_large:
-    <<: *executor-mac-arm64
+    resource_class: macos.m1.large.gen1
 
 slack-fail-stop-step: &slack-fail-post-step
   post-steps:
@@ -86,7 +70,7 @@ workflows:
           name: << matrix.platform >>_build_nightly
           matrix: &matrix-nightly
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
+              platform: ["amd64", "arm64", "mac_arm64"]
           filters: &filters-nightly
             branches:
               only:
@@ -137,7 +121,7 @@ workflows:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
           matrix:
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
+              platform: ["amd64", "arm64", "mac_arm64"]
               job_type: ["test_nightly", "integration_nightly", "e2e_expect_nightly"]
           requires:
             - << matrix.platform >>_<< matrix.job_type >>
@@ -727,12 +711,13 @@ commands:
             command: |
               if [ "${CIRCLE_BRANCH}" = "rel/nightly" ]
               then
-                export NO_BUILD="true"
+                export NIGHTLY_BUILD="true"
               fi
               export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g')
               export GOPATH="<< parameters.build_dir >>/go"
               export TRAVIS_BRANCH=${CIRCLE_BRANCH}
               scripts/travis/deploy_packages.sh
+            no_output_timeout: 20m
         - when:
             condition:
               equal: [ "amd64", << parameters.platform >> ]

.github/workflows/reviewdog.yml

@@ -69,7 +69,7 @@ jobs:
           cd ../../
       - name: Install reviewdog
         run: |
-          curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/v0.17.4/install.sh | sh -s
+          curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/v0.18.1/install.sh | sh -s -- v0.18.1
           reviewdog --version
       - name: Build custom linters
         run: |
@@ -102,3 +102,16 @@ jobs:
         run: |
           curl -X POST --data-urlencode "payload={\"text\": \"Reviewdog failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
         if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}
+  reviewdog-shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: shellcheck
+        uses: reviewdog/action-shellcheck@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          reporter: "github-pr-check"
+          shellcheck_flags: "-e SC2034,SC2046,SC2053,SC2207,SC2145 -S warning"
+          fail_on_error: true
+          path: |
+            test/scripts/e2e_subs

Makefile

@@ -11,6 +11,13 @@ endif
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
+# overrides for cross-compiling platform-specific binaries
+ifdef CROSS_COMPILE_ARCH
+  ARCH := $(CROSS_COMPILE_ARCH)
+  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOPATH1)/bin-$(OS_TYPE)-$(ARCH)
+else
+  GO_INSTALL := go install
+endif
 S3_RELEASE_BUCKET = $$S3_RELEASE_BUCKET
 
 GOLANG_VERSIONS				:= $(shell ./scripts/get_golang_version.sh all)
@@ -42,8 +49,13 @@ else
 export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
 endif
 
-# M1 Mac--homebrew install location in /opt/homebrew
 ifeq ($(OS_TYPE), darwin)
+# For Xcode >= 15, set -no_warn_duplicate_libraries linker option
+CLANG_MAJOR_VERSION := $(shell clang --version | grep '^Apple clang version ' | awk '{print $$4}' | cut -d. -f1)
+ifeq ($(shell [ $(CLANG_MAJOR_VERSION) -ge 15 ] && echo true), true)
+EXTLDFLAGS := -Wl,-no_warn_duplicate_libraries
+endif
+# M1 Mac--homebrew install location in /opt/homebrew
 ifeq ($(ARCH), arm64)
 export CPATH=/opt/homebrew/include
 export LIBRARY_PATH=/opt/homebrew/lib
@@ -102,6 +114,9 @@ fix: build
 lint: deps
 	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
 
+expectlint:
+	cd test/e2e-go/cli/goal/expect && python3 expect_linter.py *.exp
+
 check_go_version:
 	@if [ $(CURRENT_GO_VERSION_MAJOR) != $(GOLANG_VERSION_BUILD_MAJOR) ]; then \
 		echo "Wrong major version of Go installed ($(CURRENT_GO_VERSION_MAJOR)). Please use $(GOLANG_VERSION_BUILD_MAJOR)"; \
@@ -153,10 +168,40 @@ crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 	cp -R crypto/libsodium-fork/. crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork
 	cd crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork && \
 		./autogen.sh --prefix $(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH) && \
-		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" && \
+		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" $(EXTRA_CONFIGURE_FLAGS) && \
 		$(MAKE) && \
 		$(MAKE) install
 
+universal:
+ifeq ($(OS_TYPE),darwin)
+	# build amd64 Mac binaries
+	mkdir -p $(GOPATH1)/bin-darwin-amd64
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+
+	# build arm64 Mac binaries
+	mkdir -p $(GOPATH1)/bin-darwin-arm64
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
+	# same for buildsrc-special
+	cd tools/block-generator && \
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
+	# lipo together
+	mkdir -p $(GOPATH1)/bin
+	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
+		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
+			lipo -create -output $(GOPATH1)/bin/$$binary \
+			$(GOPATH1)/bin-darwin-arm64/$$binary \
+			$(GOPATH1)/bin-darwin-amd64/$$binary; \
+		else \
+			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
+		fi \
+	done
+else
+	echo "OS_TYPE must be darwin for universal builds, skipping"
+endif
+
 deps:
 	./scripts/check_deps.sh
 
@@ -212,11 +257,11 @@ ${GOCACHE}/file.txt:
 	touch "${GOCACHE}"/file.txt
 
 buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	$(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 buildsrc-special:
 	cd tools/block-generator && \
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	$(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 check-go-version:
 	./scripts/check_golang_version.sh build
@@ -331,7 +376,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger
+.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

agreement/demux.go

@@ -25,6 +25,7 @@ import (
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/logging/logspec"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/util"
 )
 
 const (
@@ -113,6 +114,7 @@ func (d *demux) tokenizeMessages(ctx context.Context, net Network, tag protocol.
 		defer func() {
 			close(decoded)
 		}()
+		util.SetGoroutineLabels("tokenizeTag", string(tag))
 		for {
 			select {
 			case raw, ok := <-networkMessages:

agreement/fuzzer/networkFacade_test.go

@@ -70,9 +70,16 @@ type NetworkFacade struct {
 	rand                           *rand.Rand
 	timeoutAtInitOnce              sync.Once
 	timeoutAtInitWait              sync.WaitGroup
-	peerToNode                     map[network.Peer]int
+	peerToNode                     map[*facadePeer]int
 }
 
+type facadePeer struct {
+	id  int
+	net network.GossipNode
+}
+
+func (p *facadePeer) GetNetwork() network.GossipNode { return p.net }
+
 // MakeNetworkFacade creates a facade with a given nodeID.
 func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade {
 	n := &NetworkFacade{
@@ -83,12 +90,12 @@ func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade {
 		eventsQueues:   make(map[string]int),
 		eventsQueuesCh: make(chan int, 1000),
 		rand:           rand.New(rand.NewSource(int64(nodeID))),
-		peerToNode:     make(map[network.Peer]int, fuzzer.nodesCount),
+		peerToNode:     make(map[*facadePeer]int, fuzzer.nodesCount),
 		debugMessages:  false,
 	}
 	n.timeoutAtInitWait.Add(1)
 	for i := 0; i < fuzzer.nodesCount; i++ {
-		n.peerToNode[network.Peer(new(int))] = i
+		n.peerToNode[&facadePeer{id: i, net: n}] = i
 	}
 	return n
 }
@@ -179,7 +186,7 @@ func (n *NetworkFacade) WaitForEventsQueue(cleared bool) {
 func (n *NetworkFacade) Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, exclude network.Peer) error {
 	excludeNode := -1
 	if exclude != nil {
-		excludeNode = n.peerToNode[exclude]
+		excludeNode = n.peerToNode[exclude.(*facadePeer)]
 	}
 	return n.broadcast(tag, data, excludeNode, "NetworkFacade service-%v Broadcast %v %v\n")
 }
@@ -240,7 +247,7 @@ func (n *NetworkFacade) PushDownstreamMessage(newMsg context.CancelFunc) bool {
 func (n *NetworkFacade) Address() (string, bool) { return "mock network", true }
 
 // Start - unused function
-func (n *NetworkFacade) Start() {}
+func (n *NetworkFacade) Start() error { return nil }
 
 // Stop - unused function
 func (n *NetworkFacade) Stop() {}
@@ -341,8 +348,8 @@ func (n *NetworkFacade) ReceiveMessage(sourceNode int, tag protocol.Tag, data []
 	n.pushPendingReceivedMessage()
 }
 
-func (n *NetworkFacade) Disconnect(sender network.Peer) {
-	sourceNode := n.peerToNode[sender]
+func (n *NetworkFacade) Disconnect(sender network.DisconnectablePeer) {
+	sourceNode := n.peerToNode[sender.(*facadePeer)]
 	n.fuzzer.Disconnect(n.nodeID, sourceNode)
 }
 

agreement/gossip/network_test.go

@@ -18,7 +18,6 @@ package gossip
 
 import (
 	"context"
-	"net"
 	"net/http"
 	"sync"
 	"sync/atomic"
@@ -136,7 +135,7 @@ func (w *whiteholeNetwork) Relay(ctx context.Context, tag protocol.Tag, data []b
 func (w *whiteholeNetwork) BroadcastSimple(tag protocol.Tag, data []byte) error {
 	return w.Broadcast(context.Background(), tag, data, true, nil)
 }
-func (w *whiteholeNetwork) Disconnect(badnode network.Peer) {
+func (w *whiteholeNetwork) Disconnect(badnode network.DisconnectablePeer) {
 	return
 }
 func (w *whiteholeNetwork) DisconnectPeers() {
@@ -156,11 +155,8 @@ func (w *whiteholeNetwork) GetPeers(options ...network.PeerOption) []network.Pee
 }
 func (w *whiteholeNetwork) RegisterHTTPHandler(path string, handler http.Handler) {
 }
-func (w *whiteholeNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) {
-	return nil
-}
 
-func (w *whiteholeNetwork) Start() {
+func (w *whiteholeNetwork) Start() error {
 	w.quit = make(chan struct{})
 	go func(w *whiteholeNetwork) {
 		w.domain.messagesMu.Lock()
@@ -216,7 +212,7 @@ func (w *whiteholeNetwork) Start() {
 			atomic.AddUint32(&w.lastMsgRead, 1)
 		}
 	}(w)
-	return
+	return nil
 }
 func (w *whiteholeNetwork) getMux() *network.Multiplexer {
 	return w.mux

agreement/service.go

@@ -173,6 +173,9 @@ func (s *Service) Start() {
 //
 // This method returns after all resources have been cleaned up.
 func (s *Service) Shutdown() {
+	s.log.Debug("agreement service is stopping")
+	defer s.log.Debug("agreement service has stopped")
+
 	close(s.quit)
 	s.quitFn()
 	<-s.done

catchup/catchpointService.go

@@ -185,6 +185,9 @@ func (cs *CatchpointCatchupService) Abort() {
 // Stop stops the catchpoint catchup service - unlike Abort, this is not intended to abort the process but rather to allow
 // cleanup of in-memory resources for the purpose of clean shutdown.
 func (cs *CatchpointCatchupService) Stop() {
+	cs.log.Debug("catchpoint service is stopping")
+	defer cs.log.Debug("catchpoint service has stopped")
+
 	// signal the running goroutine that we want to stop
 	cs.cancelCtxFunc()
 	// wait for the running goroutine to terminate.