Capling: long-living `rotx` in blob_downloader

[AskAlexSharov]

gnosis, 3.4:

[dbg.db.caplin] long living resources    list="36875(25m24.605824076s): [kv_mdbx.go:618 blob_downloader.go:213 blob_downloader.go:167]"

18G	/erigon-data/caplin/indexing

./build/bin/mdbx_stat -efa  /erigon-data/caplin/indexing | awk '
    BEGIN { pagesize = 4096 }
    /^  Pagesize:/ { pagesize = $2 }
    /^Status of/ { table = $3 }
    /Branch pages:/ { branch = $3 }
    /Leaf pages:/ { leaf = $3 }
    /Overflow pages:/ { overflow = $3 }
    /Entries:/ {
      table_pages = branch + leaf + overflow
      size_gb = (table_pages * pagesize) / (1024^3)
      printf "%-30s %.2fG\n", table, size_gb
    }
    /Reclaimable:/ {
      table = "Reclaimable"
      table_pages = $2
      size_gb = (table_pages * pagesize) / (1024^3)
      printf "%-30s %.2fG\n", table, size_gb
    }
  ' | grep -v '0.0'
   open-MADV_DONTNEED 4718592..4718592
   readahead OFF 0..4718592
BeaconBlock                    10.11G
BeaconBlockHeaders             0.87G
BlockRootToBlockHash           0.25G
BlockRootToBlockNumber         0.15G
BlockRootToParentRoot          0.25G
BlockRootToSlot                0.15G
BlockRootToStateRoot           0.25G
CanonicalBlockRoots            0.13G
ParentRootToBlockRoots         0.25G
StateRootToBlockRoot           0.25G

Goroutines:

1 @ 0x4bbacb 0x1450e46 0x14588b1 0x1458844 0x14633a6 0x1b5dafe 0x1b5da4a 0x1fb1f06 0x4c7901
#	0x1450e45	github.com/erigontech/mdbx-go/mdbx._Cfunc_mdbxgo_put2+0x45				_cgo_gotypes.go:1410
#	0x14588b0	github.com/erigontech/mdbx-go/mdbx.(*Txn).Put.func1+0xd0				github.com/erigontech/mdbx-go@v0.39.17/mdbx/txn.go:678
#	0x1458843	github.com/erigontech/mdbx-go/mdbx.(*Txn).Put+0x63					github.com/erigontech/mdbx-go@v0.39.17/mdbx/txn.go:678
#	0x14633a5	github.com/erigontech/erigon/db/kv/mdbx.(*MdbxTx).Put+0xa5				github.com/erigontech/erigon/db/kv/mdbx/kv_mdbx.go:1100
#	0x1b5dafd	github.com/erigontech/erigon/cl/persistence/beacon_indicies.WriteParentBlockRoot+0x8dd	github.com/erigontech/erigon/cl/persistence/beacon_indicies/indicies.go:244
#	0x1b5da49	github.com/erigontech/erigon/cl/antiquary.(*Antiquary).Loop+0x829			github.com/erigontech/erigon/cl/antiquary/antiquary.go:203
#	0x1fb1f05	github.com/erigontech/erigon/cmd/caplin/caplin1.RunCaplinService.func3+0xe5		github.com/erigontech/erigon/cmd/caplin/caplin1/run.go:470


1 @ 0x4bbacb 0x144f606 0x1455eb3 0x1455e2a 0x146006d 0x1460053 0x1461dc5 0x1461dc6 0x1f8e3c8 0x1f93a45 0x1f94c30 0x1f8875a 0x1f94846 0x1f98751 0x4c7901
#	0x144f605	github.com/erigontech/mdbx-go/mdbx._Cfunc_mdbx_txn_begin+0x45					_cgo_gotypes.go:1053
#	0x1455eb2	github.com/erigontech/mdbx-go/mdbx.beginTxn.func1+0x152						github.com/erigontech/mdbx-go@v0.39.17/mdbx/txn.go:105
#	0x1455e29	github.com/erigontech/mdbx-go/mdbx.beginTxn+0xc9						github.com/erigontech/mdbx-go@v0.39.17/mdbx/txn.go:105
#	0x146006c	github.com/erigontech/mdbx-go/mdbx.(*Env).BeginTxn+0x8c						github.com/erigontech/mdbx-go@v0.39.17/mdbx/env.go:531
#	0x1460052	github.com/erigontech/erigon/db/kv/mdbx.(*MdbxKV).beginRw+0x72					github.com/erigontech/erigon/db/kv/mdbx/kv_mdbx.go:641
#	0x1461dc4	github.com/erigontech/erigon/db/kv/mdbx.(*MdbxKV).BeginRw+0x44					github.com/erigontech/erigon/db/kv/mdbx/kv_mdbx.go:623
#	0x1461dc5	github.com/erigontech/erigon/db/kv/mdbx.(*MdbxKV).Update+0x45					github.com/erigontech/erigon/db/kv/mdbx/kv_mdbx.go:897
#	0x1f8e3c7	github.com/erigontech/erigon/cl/phase1/stages.processBlock+0xe7					github.com/erigontech/erigon/cl/phase1/stages/clstages.go:163
#	0x1f93a44	github.com/erigontech/erigon/cl/phase1/stages.processDownloadedBlockBatches+0x304		github.com/erigontech/erigon/cl/phase1/stages/forward_sync.go:182
#	0x1f94c2f	github.com/erigontech/erigon/cl/phase1/stages.forwardSync.func1+0x6f				github.com/erigontech/erigon/cl/phase1/stages/forward_sync.go:263
#	0x1f88759	github.com/erigontech/erigon/cl/phase1/network.(*ForwardBeaconDownloader).RequestMore+0x3f9	github.com/erigontech/erigon/cl/phase1/network/beacon_downloader.go:151
#	0x1f94845	github.com/erigontech/erigon/cl/phase1/stages.forwardSync+0x465					github.com/erigontech/erigon/cl/phase1/stages/forward_sync.go:279
#	0x1f98750	github.com/erigontech/erigon/cl/clstages.(*StageGraph[...]).StartWithStage.func1+0x70		github.com/erigontech/erigon/cl/clstages/clstages.go:54

I also see time.Sleep:

if !b.syncedChecker.Synced() {
			time.Sleep(5 * time.Second)
			continue
		}

and

func (s *Sentinel) stickToPeers(peers []multiaddr.Multiaddr) {
	// connect to static peers every one minute
	go func() {
		for {
			if err := s.connectWithAllPeers(peers); err != nil {
				log.Debug("[Sentinel] Could not connect with static peers", "err", err)
			}
			time.Sleep(3 * time.Minute)
		}
	}()
}

and

func (r *tokenBucketRateLimiter) cleanup() {
	for {
		time.Sleep(10 * time.Second)
		r.mu.Lock()
		for key, limiter := range r.limiter {
			if time.Since(limiter.lastAccess) > 3*time.Minute {
				delete(r.limiter, key)
			}
		}
		r.mu.Unlock()
	}
}