Merge remote-tracking branch 'upstream/master'

Author: actions-user

configs/coins/zcash.json

@@ -22,10 +22,10 @@
         "package_name": "backend-zcash",
         "package_revision": "satoshilabs-1",
         "system_user": "zcash",
-        "version": "4.4.1",
-        "docker_image": "zfnd/zebra:4.4.1",
+        "version": "4.5.0",
+        "docker_image": "zfnd/zebra:4.5.0",
         "verification_type": "docker",
-        "verification_source": "96149af0257d1f52612544b68f160f8c1bd1d229a47aced203bfa35f4925137d",
+        "verification_source": "9fd0125f01a04f3ebaec3d0c6426ee4fa48c7a587e62a12d1ac79c3d5fc357e7",
         "extract_command": "mkdir backend/bin && docker cp extract:/usr/local/bin/zebrad backend/bin/zebrad",
         "exclude_files": [],
         "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zebrad --config {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/zcash.conf start",

db/address_hotness.go

@@ -32,9 +32,14 @@ type addressHotness struct {
 	minHits      int
 	lru          *hotAddressLRU
 	onEvict      func(addressHotnessKey)
-	// hits tracks per-block lookup counts so we can decide when an address is hot.
-	// It is cleared at BeginBlock to avoid unbounded growth.
+	// hits tracks lookup counts so we can decide when an address is hot. Counts
+	// accumulate across blocks so an address that recurs over several blocks (not
+	// only within one busy block) can become hot; the map is reset in BeginBlock
+	// once it grows past maxPendingHits to keep memory bounded.
 	hits map[addressHotnessKey]uint16
+	// maxPendingHits bounds the hits map (set to the LRU size, the natural ceiling
+	// for promotion candidates).
+	maxPendingHits int
 	// block stats (reset after reporting) to keep logging cheap.
 	// blockEligibleLookups counts lookups with contractCount >= minContracts (i.e., eligible for hotness).
 	blockEligibleLookups uint64
@@ -51,11 +56,11 @@ func newAddressHotness(minContracts, lruSize, minHits int) *addressHotness {
 		return nil
 	}
 	return &addressHotness{
-		minContracts: minContracts,
-		minHits:      minHits,
-		lru:          newHotAddressLRU(lruSize),
-		// Pre-size the per-block hit map to avoid reallocs on busy blocks.
-		hits: make(map[addressHotnessKey]uint16),
+		minContracts:   minContracts,
+		minHits:        minHits,
+		lru:            newHotAddressLRU(lruSize),
+		maxPendingHits: lruSize,
+		hits:           make(map[addressHotnessKey]uint16),
 	}
 }
 
@@ -72,9 +77,20 @@ func (h *addressHotness) BeginBlock() {
 	if h == nil {
 		return
 	}
-	// Reset per-block hit counts; LRU survives across blocks.
-	clear(h.hits)
-	// Reset per-block stats counters.
+	// Hit counts accumulate across blocks so addresses looked up repeatedly over
+	// several blocks (not only within one busy block) can become hot — this lets
+	// the index help lower-throughput chains, not just very busy ones. Reset only
+	// when the candidate map grows past its bound; dropping pending counts merely
+	// delays a promotion and never affects correctness (lookups fall back to a
+	// linear scan when the index is not used). The LRU survives across blocks.
+	if len(h.hits) > h.maxPendingHits {
+		// Reinitialize rather than clear(): Go's clear() does not shrink a map's
+		// underlying bucket allocation, so a single oversized block would leave the
+		// allocation at its high-water mark. Pre-size to maxPendingHits, the
+		// steady-state ceiling, so the oversized buckets can be released.
+		h.hits = make(map[addressHotnessKey]uint16, h.maxPendingHits)
+	}
+	// Reset per-block stats counters (metrics report per-block deltas).
 	h.blockEligibleLookups = 0
 	h.blockLRUHits = 0
 	h.blockPromotions = 0

db/address_hotness_test.go

@@ -122,7 +122,7 @@ func Test_addressHotness_LRUEvictionHook(t *testing.T) {
 }
 
 func Test_addressHotness_Specs(t *testing.T) {
-	t.Run("it should reset per-block hits", func(t *testing.T) {
+	t.Run("it should accumulate hits across blocks", func(t *testing.T) {
 		hot := newAddressHotness(1, 2, 2)
 		if hot == nil {
 			t.Fatal("expected hotness tracker to be initialized")
@@ -133,8 +133,30 @@ func Test_addressHotness_Specs(t *testing.T) {
 			t.Fatal("expected first hit to stay cold")
 		}
 		hot.BeginBlock()
-		if hot.ShouldUseIndex(key, 1) {
-			t.Fatal("expected hit count to reset between blocks")
+		if !hot.ShouldUseIndex(key, 1) {
+			t.Fatal("expected hit counts to accumulate across blocks and promote")
+		}
+	})
+
+	t.Run("it should reset pending hits once the candidate map exceeds its bound", func(t *testing.T) {
+		// lruSize (and thus maxPendingHits) is 1 here.
+		hot := newAddressHotness(1, 1, 2)
+		if hot == nil {
+			t.Fatal("expected hotness tracker to be initialized")
+		}
+		a := makeHotKey(30)
+		b := makeHotKey(31)
+		hot.BeginBlock()
+		if hot.ShouldUseIndex(a, 1) {
+			t.Fatal("expected first hit on A to stay cold")
+		}
+		if hot.ShouldUseIndex(b, 1) {
+			t.Fatal("expected first hit on B to stay cold")
+		}
+		// hits now holds 2 pending entries > maxPendingHits (1), so BeginBlock clears it.
+		hot.BeginBlock()
+		if hot.ShouldUseIndex(a, 1) {
+			t.Fatal("expected A's pending hit to be cleared once the bound was exceeded")
 		}
 	})
 

db/sync.go

@@ -43,10 +43,11 @@ type MissingBlockRetryConfig struct {
 	TipRecheckThreshold int
 	// RetryDelay keeps retry pressure low while still reacting quickly to transient backend gaps.
 	RetryDelay time.Duration
-	// MaxStallDuration caps the wall-clock time a single block fetch may spend
-	// in the retry loop before yielding control to the outer resync machinery.
-	// Without this cap, a backend that keeps returning ErrBlockNotFound while
-	// shouldRestartSyncOnMissingBlock keeps reporting "no reorg" loops forever.
+	// MaxStallDuration caps the wall-clock time a single block fetch may spend in
+	// the retry loop before yielding errResync. Liveness invariant: since lagging
+	// probes report "no reorg" and known hashes get retried, a genuinely-behind
+	// backend or chain-shortening reorg relies on this cap. Must stay > 0
+	// (ApplyMissingBlockRetryOverride enforces it).
 	MaxStallDuration time.Duration
 }
 
@@ -118,6 +119,14 @@ func NewSyncWorkerWithConfig(db *RocksDB, chain bchain.BlockChain, syncWorkers,
 	if cfg != nil {
 		effectiveCfg = *cfg
 	}
+	// MaxStallDuration is the load-bearing liveness cap (see its doc): the retry
+	// loops disable the cap when it's <= 0, which would let a chain-shortening
+	// reorg spin forever. Enforce the invariant structurally here so every caller
+	// (including tests passing a partial cfg) gets a safe value, not just the
+	// ApplyMissingBlockRetryOverride path.
+	if effectiveCfg.MissingBlockRetry.MaxStallDuration <= 0 {
+		effectiveCfg.MissingBlockRetry.MaxStallDuration = DefaultMissingBlockRetryConfig().MaxStallDuration
+	}
 	return &SyncWorker{
 		db:                db,
 		chain:             chain,
@@ -255,44 +264,44 @@ func (w *SyncWorker) resyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync b
 			return err
 		}
 		if remoteBestHeight < w.startHeight {
-			glog.Error("resync: error - remote best height ", remoteBestHeight, " less than sync start height ", w.startHeight)
-			return errors.New("resync: remote best height error")
-		}
-		if initialSync {
-			if remoteBestHeight-w.startHeight > uint32(w.syncChunk) {
-				glog.Infof("resync: bulk sync of blocks %d-%d, using %d workers", w.startHeight, remoteBestHeight, w.syncWorkers)
-				// Bulk sync can encounter a disappearing block hash during reorgs.
-				// When that happens, it returns errResync to trigger a full restart.
-				err = w.BulkConnectBlocks(w.startHeight, remoteBestHeight)
-				if err != nil {
-					if stdErrors.Is(err, errResync) {
-						// block hash changed during parallel sync, restart the full resync
-						return w.resyncIndex(onNewBlock, initialSync)
+			glog.Warning("resync: observed remote best height ", remoteBestHeight, " less than sync start height ", w.startHeight, ", falling back to sequential sync")
+		} else {
+			if initialSync {
+				if remoteBestHeight-w.startHeight > uint32(w.syncChunk) {
+					glog.Infof("resync: bulk sync of blocks %d-%d, using %d workers", w.startHeight, remoteBestHeight, w.syncWorkers)
+					// Bulk sync can encounter a disappearing block hash during reorgs.
+					// When that happens, it returns errResync to trigger a full restart.
+					err = w.BulkConnectBlocks(w.startHeight, remoteBestHeight)
+					if err != nil {
+						if stdErrors.Is(err, errResync) {
+							// block hash changed during parallel sync, restart the full resync
+							return w.resyncIndex(onNewBlock, initialSync)
+						}
+						return err
 					}
-					return err
+					// after parallel load finish the sync using standard way,
+					// new blocks may have been created in the meantime
+					return w.resyncIndex(onNewBlock, initialSync)
 				}
-				// after parallel load finish the sync using standard way,
-				// new blocks may have been created in the meantime
-				return w.resyncIndex(onNewBlock, initialSync)
 			}
-		}
-		if w.chain.GetChainParser().GetChainType() == bchain.ChainEthereumType {
-			syncWorkers := uint32(4)
-			if remoteBestHeight-w.startHeight >= syncWorkers {
-				glog.Infof("resync: parallel sync of blocks %d-%d, using %d workers", w.startHeight, remoteBestHeight, syncWorkers)
-				// Parallel sync also returns errResync when a requested hash no longer
-				// exists at its height; restart to realign with the canonical chain.
-				err = w.ParallelConnectBlocks(onNewBlock, w.startHeight, remoteBestHeight, syncWorkers)
-				if err != nil {
-					if stdErrors.Is(err, errResync) {
-						// block hash changed during parallel sync, restart the full resync
-						return w.resyncIndex(onNewBlock, initialSync)
+			if w.chain.GetChainParser().GetChainType() == bchain.ChainEthereumType {
+				syncWorkers := uint32(4)
+				if remoteBestHeight-w.startHeight >= syncWorkers {
+					glog.Infof("resync: parallel sync of blocks %d-%d, using %d workers", w.startHeight, remoteBestHeight, syncWorkers)
+					// Parallel sync also returns errResync when a requested hash no longer
+					// exists at its height; restart to realign with the canonical chain.
+					err = w.ParallelConnectBlocks(onNewBlock, w.startHeight, remoteBestHeight, syncWorkers)
+					if err != nil {
+						if stdErrors.Is(err, errResync) {
+							// block hash changed during parallel sync, restart the full resync
+							return w.resyncIndex(onNewBlock, initialSync)
+						}
+						return err
 					}
-					return err
+					// after parallel load finish the sync using standard way,
+					// new blocks may have been created in the meantime
+					return w.resyncIndex(onNewBlock, initialSync)
 				}
-				// after parallel load finish the sync using standard way,
-				// new blocks may have been created in the meantime
-				return w.resyncIndex(onNewBlock, initialSync)
 			}
 		}
 	}
@@ -400,21 +409,37 @@ type hashHeight struct {
 	height uint32
 }
 
+// sendHashHeight queues hh but stays abort-aware: if a full hch made this a blocking
+// send, the coordinator could never read abortCh and sync would wedge. On abort hh is
+// intentionally dropped since the round is being torn down anyway.
+func (w *SyncWorker) sendHashHeight(hch chan<- hashHeight, abortCh <-chan error, hh hashHeight) error {
+	select {
+	case hch <- hh:
+		return nil
+	case abortErr := <-abortCh:
+		return abortErr
+	case <-w.chanOsSignal:
+		return ErrOperationInterrupted
+	}
+}
+
 func (w *SyncWorker) shouldRestartSyncOnMissingBlock(height uint32, expectedHash string) (bool, error) {
-	// When a block hash disappears at a given height, it usually indicates a
-	// reorg/rollback. Confirm by checking the current tip and block hash.
+	// When a block hash disappears at a given height, it can indicate a
+	// reorg/rollback, but on load-balanced EVM RPCs a single lagging backend can
+	// also report an older tip. Only restart immediately when another probe can
+	// prove the height exists with a different hash; otherwise let the retry
+	// loop or wall-clock cap yield control to the outer resync.
 	bestHeight, err := w.chain.GetBestBlockHeight()
 	if err != nil {
 		return false, err
 	}
 	if bestHeight < height {
-		// The tip moved below the requested height, so this block is no longer valid.
-		return true, nil
+		return false, nil
 	}
 	currentHash, err := w.chain.GetBlockHash(height)
 	if err != nil {
 		if stdErrors.Is(err, bchain.ErrBlockNotFound) {
-			return true, nil
+			return false, nil
 		}
 		return false, err
 	}
@@ -574,7 +599,17 @@ ConnectLoop:
 				time.Sleep(time.Millisecond * 500)
 				continue
 			}
-			hch <- hashHeight{hash, h}
+			if err = w.sendHashHeight(hch, abortCh, hashHeight{hash, h}); err != nil {
+				if stdErrors.Is(err, errResync) {
+					glog.Warning("sync: parallel connect aborted while queueing block hash, restarting sync")
+				} else if stdErrors.Is(err, ErrOperationInterrupted) {
+					glog.Info("connectBlocksParallel interrupted at height ", h)
+				} else {
+					glog.Error("sync: parallel connect aborted while queueing block hash, worker error ", err)
+				}
+				close(terminating)
+				break ConnectLoop
+			}
 			h++
 		}
 	}
@@ -791,7 +826,7 @@ ConnectLoop:
 			close(terminating)
 			break ConnectLoop
 		case <-w.chanOsSignal:
-			glog.Info("connectBlocksParallel interrupted at height ", h)
+			glog.Info("BulkConnectBlocks interrupted at height ", h)
 			err = ErrOperationInterrupted
 			// signal all workers to terminate their loops (error loops are interrupted below)
 			close(terminating)
@@ -804,7 +839,17 @@ ConnectLoop:
 				time.Sleep(time.Millisecond * 500)
 				continue
 			}
-			hch <- hashHeight{hash, h}
+			if err = w.sendHashHeight(hch, abortCh, hashHeight{hash, h}); err != nil {
+				if stdErrors.Is(err, errResync) {
+					glog.Warning("sync: bulk connect aborted while queueing block hash, restarting sync")
+				} else if stdErrors.Is(err, ErrOperationInterrupted) {
+					glog.Info("BulkConnectBlocks interrupted at height ", h)
+				} else {
+					glog.Error("sync: bulk connect aborted while queueing block hash, worker error ", err)
+				}
+				close(terminating)
+				break ConnectLoop
+			}
 			if h > 0 && h%1000 == 0 {
 				w.metrics.BlockbookBestHeight.Set(float64(h))
 				glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start), " ", w.db.GetAndResetConnectBlockStats())
@@ -891,7 +936,10 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) {
 					return
 				}
 				if height > bestHeight {
-					return
+					if hash == "" {
+						return
+					}
+					glog.Warningf("getBlockChain: block %d %s is above observed backend height %d; retrying because the block hash was already observed", height, hash, bestHeight)
 				}
 			}
 			if gotNotFound {