Merge pull request #2724 from ledgerwatch/stable-2021-09-04

Stable 2021 09 04

Author: mandrigin

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-16.04, ubuntu-20.04, macos-10.15 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-20.04, macos-10.15 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:

README.md

@@ -180,7 +180,7 @@ Erigon uses a rearchitected full sync algorithm from
 It uses the same network primitives and is compatible with regular go-ethereum nodes that are using full sync, you do
 not need any special sync capabilities for Erigon to sync.
 
-When reimagining the full sync, we focused on batching data together and minimize DB overwrites. That makes it possible
+When reimagining the full sync, with focus on batching data together and minimize DB overwrites. That makes it possible
 to sync Ethereum mainnet in under 2 days if you have a fast enough network connection and an SSD drive.
 
 Examples of stages are:
@@ -316,6 +316,14 @@ you'll have to change one if you want to run both at the same time. use `--help`
 
 Reserved for future use: **gRPC ports**: `9092` consensus engine, `9093` snapshot downloader, `9094` TxPool
 
+### How to get diagnostic for bug report?
+
+- Get stack trace: `kill -SIGUSR1 <pid>`, get trace and stop: `kill -6 <pid>`
+- Get CPU profiling: add `--pprof flag`
+  run `go tool pprof -png  http://127.0.0.1:6060/debug/pprof/profile\?seconds\=20 > cpu.png`
+- Get RAM profiling: add `--pprof flag`
+  run `go tool pprof -inuse_space -png  http://127.0.0.1:6060/debug/pprof/heap > mem.png`
+
 Getting in touch
 ================
 
@@ -405,3 +413,7 @@ non-batched way.
 ### Filesystem's background features are expensive
 
 For example: btrfs's autodefrag option - may increase write IO 100x times
+
+### Gnome Tracker can kill Erigon
+
+[Gnome Tracker](https://wiki.gnome.org/Projects/Tracker) - detecting miners and kill them.

cmd/cons/commands/clique.go

@@ -124,6 +124,12 @@ func grpcCliqueServer(ctx context.Context, testServer bool) (*CliqueServerImpl,
 		grpc.KeepaliveParams(keepalive.ServerParameters{
 			Time: 10 * time.Minute,
 		}),
+		// Don't drop the connection, settings accordign to this comment on GitHub
+		// https://github.com/grpc/grpc-go/issues/3171#issuecomment-552796779
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime:             10 * time.Second,
+			PermitWithoutStream: true,
+		}),
 		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)),
 		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),
 	}

cmd/hack/hack.go

@@ -22,6 +22,7 @@ import (
 	"github.com/holiman/uint256"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/mdbx"
+	"github.com/ledgerwatch/erigon-lib/recsplit"
 	"github.com/ledgerwatch/erigon/consensus/ethash"
 	"github.com/ledgerwatch/erigon/consensus/misc"
 	"github.com/ledgerwatch/erigon/core"
@@ -887,19 +888,6 @@ func validateTxLookups2(db kv.RwDB, startBlock uint64, interruptCh chan bool) {
 	}
 }
 
-func getModifiedAccounts(chaindata string) {
-	// TODO(tjayrush): The call to GetModifiedAccounts needs a database tx
-	fmt.Println("hack - getModiiedAccounts is temporarily disabled.")
-	db := mdbx.MustOpen(chaindata)
-	defer db.Close()
-	tool.Check(db.View(context.Background(), func(tx kv.Tx) error {
-		addrs, err := changeset.GetModifiedAccounts(tx, 49300, 49400)
-		tool.Check(err)
-		fmt.Printf("Len(addrs)=%d\n", len(addrs))
-		return nil
-	}))
-}
-
 type Receiver struct {
 	defaultReceiver *trie.RootHashAggregator
 	accountMap      map[string]*accounts.Account
@@ -1011,52 +999,44 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
 	log.Info("GetProof", "address", address, "storage keys", len(storageKeys), "head", *headNumber, "block", block,
 		"alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
 
-	ts := dbutils.EncodeBlockNumber(block + 1)
 	accountMap := make(map[string]*accounts.Account)
 
-	if err := changeset.Walk(tx, kv.AccountChangeSet, ts, 0, func(blockN uint64, address, v []byte) (bool, error) {
-		if blockN > *headNumber {
-			return false, nil
-		}
-
+	if err := changeset.ForRange(tx, kv.AccountChangeSet, block+1, *headNumber+1, func(blockN uint64, address, v []byte) error {
 		var addrHash, err = common.HashData(address)
 		if err != nil {
-			return false, err
+			return err
 		}
 		k := addrHash[:]
 
 		if _, ok := accountMap[string(k)]; !ok {
 			if len(v) > 0 {
 				var a accounts.Account
 				if innerErr := a.DecodeForStorage(v); innerErr != nil {
-					return false, innerErr
+					return innerErr
 				}
 				accountMap[string(k)] = &a
 			} else {
 				accountMap[string(k)] = nil
 			}
 		}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
 	runtime.ReadMemStats(&m)
 	log.Info("Constructed account map", "size", len(accountMap),
 		"alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
 	storageMap := make(map[string][]byte)
-	if err := changeset.Walk(tx, kv.StorageChangeSet, ts, 0, func(blockN uint64, address, v []byte) (bool, error) {
-		if blockN > *headNumber {
-			return false, nil
-		}
+	if err := changeset.ForRange(tx, kv.StorageChangeSet, block+1, *headNumber+1, func(blockN uint64, address, v []byte) error {
 		var addrHash, err = common.HashData(address)
 		if err != nil {
-			return false, err
+			return err
 		}
 		k := addrHash[:]
 		if _, ok := storageMap[string(k)]; !ok {
 			storageMap[string(k)] = v
 		}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
@@ -1137,7 +1117,7 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
 	return nil
 }
 
-func dumpState(chaindata string) error {
+func dumpState(chaindata string, block uint64) error {
 	db := mdbx.MustOpen(chaindata)
 	defer db.Close()
 	f, err := os.Create("statedump")
@@ -1149,39 +1129,91 @@ func dumpState(chaindata string) error {
 	defer w.Flush()
 	stAccounts := 0
 	stStorage := 0
-	var varintBuf [10]byte // Buffer for varint number
+	var rs *recsplit.RecSplit
 	if err := db.View(context.Background(), func(tx kv.Tx) error {
 		c, err := tx.Cursor(kv.PlainState)
 		if err != nil {
 			return err
 		}
-		k, v, e := c.First()
-		for ; k != nil && e == nil; k, v, e = c.Next() {
-			keyLen := binary.PutUvarint(varintBuf[:], uint64(len(k)))
-			if _, err = w.Write(varintBuf[:keyLen]); err != nil {
+		var count uint64
+		if count, err = c.Count(); err != nil {
+			return err
+		}
+		if block > 1 {
+			count = block
+		}
+		if rs, err = recsplit.NewRecSplit(recsplit.RecSplitArgs{
+			KeyCount:   int(count),
+			BucketSize: 2000,
+			Salt:       0,
+			LeafSize:   8,
+			TmpDir:     "",
+			StartSeed: []uint64{0x106393c187cae21a, 0x6453cec3f7376937, 0x643e521ddbd2be98, 0x3740c6412f6572cb, 0x717d47562f1ce470, 0x4cd6eb4c63befb7c, 0x9bfd8c5e18c8da73,
+				0x082f20e10092a9a3, 0x2ada2ce68d21defc, 0xe33cb4f3e7c6466b, 0x3980be458c509c59, 0xc466fd9584828e8c, 0x45f0aabe1a61ede6, 0xf6e7b8b33ad9b98d,
+				0x4ef95e25f4b4983d, 0x81175195173b92d3, 0x4e50927d8dd15978, 0x1ea2099d1fafae7f, 0x425c8a06fbaaa815, 0xcd4216006c74052a},
+		}); err != nil {
+			return err
+		}
+		k, _, e := c.First()
+		i := 0
+		for ; k != nil && e == nil; k, _, e = c.Next() {
+			if err := rs.AddKey(k); err != nil {
 				return err
 			}
-			if _, err = w.Write([]byte(k)); err != nil {
-				return err
+			i++
+			if i == int(count) {
+				break
 			}
-			valLen := binary.PutUvarint(varintBuf[:], uint64(len(v)))
-			if _, err = w.Write(varintBuf[:valLen]); err != nil {
-				return err
+			if i%1_000_000 == 0 {
+				log.Info("Added", "keys", i)
 			}
-			if len(v) > 0 {
-				if _, err = w.Write(v); err != nil {
-					return err
+		}
+		if e != nil {
+			return e
+		}
+		start := time.Now()
+		log.Info("Building recsplit...")
+		if err = rs.Build(); err != nil {
+			return err
+		}
+		s1, s2 := rs.Stats()
+		log.Info("Done", "time", time.Since(start), "s1", s1, "s2", s2)
+		log.Info("Testing bijection")
+		bitCount := (count + 63) / 64
+		bits := make([]uint64, bitCount)
+		k, _, e = c.First()
+		i = 0
+		var collisionMap map[int]string
+		if count < 1000000 {
+			collisionMap = make(map[int]string)
+		}
+		var lookupTime time.Duration
+		for ; k != nil && e == nil; k, _, e = c.Next() {
+			start := time.Now()
+			idx := rs.Lookup(k, false /* trace */)
+			lookupTime += time.Since(start)
+			if idx >= int(count) {
+				return fmt.Errorf("idx %d >= count %d", idx, count)
+			}
+			mask := uint64(1) << (idx & 63)
+			if bits[idx>>6]&mask != 0 {
+				if collisionMap != nil {
+					fmt.Printf("Key %x collided with key %x\n", k, collisionMap[idx])
 				}
+				rs.Lookup([]byte(collisionMap[idx]), true)
+				rs.Lookup(k, true)
+				return fmt.Errorf("no bijection key idx=%d, lookup up idx = %d", i, idx)
 			}
-			if len(k) > 28 {
-				stStorage++
-			} else {
-				stAccounts++
+			if collisionMap != nil {
+				collisionMap[idx] = string(k)
 			}
-			if (stStorage+stAccounts)%100000 == 0 {
-				fmt.Printf("State records: %d\n", stStorage+stAccounts)
+			bits[idx>>6] |= mask
+			i++
+			if i == int(count) {
+				break
 			}
 		}
+		log.Info("Average lookup time", "per key", time.Duration(uint64(lookupTime)/count))
 		return e
 	}); err != nil {
 		return err
@@ -1225,29 +1257,23 @@ func changeSetStats(chaindata string, block1, block2 uint64) error {
 		return err1
 	}
 	defer tx.Rollback()
-	if err := changeset.Walk(tx, kv.AccountChangeSet, dbutils.EncodeBlockNumber(block1), 0, func(blockN uint64, k, v []byte) (bool, error) {
-		if blockN >= block2 {
-			return false, nil
-		}
+	if err := changeset.ForRange(tx, kv.AccountChangeSet, block1, block2, func(blockN uint64, k, v []byte) error {
 		if (blockN-block1)%100000 == 0 {
 			fmt.Printf("at the block %d for accounts, booster size: %d\n", blockN, len(accounts))
 		}
 		accounts[string(common.CopyBytes(k))] = struct{}{}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
 
 	storage := make(map[string]struct{})
-	if err := changeset.Walk(tx, kv.StorageChangeSet, dbutils.EncodeBlockNumber(block1), 0, func(blockN uint64, k, v []byte) (bool, error) {
-		if blockN >= block2 {
-			return false, nil
-		}
+	if err := changeset.ForRange(tx, kv.StorageChangeSet, block1, block2, func(blockN uint64, k, v []byte) error {
 		if (blockN-block1)%100000 == 0 {
 			fmt.Printf("at the block %d for accounts, booster size: %d\n", blockN, len(accounts))
 		}
 		storage[string(common.CopyBytes(k))] = struct{}{}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
@@ -1266,11 +1292,11 @@ func searchChangeSet(chaindata string, key []byte, block uint64) error {
 	}
 	defer tx.Rollback()
 
-	if err := changeset.Walk(tx, kv.AccountChangeSet, dbutils.EncodeBlockNumber(block), 0, func(blockN uint64, k, v []byte) (bool, error) {
+	if err := changeset.ForEach(tx, kv.AccountChangeSet, dbutils.EncodeBlockNumber(block), func(blockN uint64, k, v []byte) error {
 		if bytes.Equal(k, key) {
 			fmt.Printf("Found in block %d with value %x\n", blockN, v)
 		}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
@@ -1286,11 +1312,11 @@ func searchStorageChangeSet(chaindata string, key []byte, block uint64) error {
 		return err1
 	}
 	defer tx.Rollback()
-	if err := changeset.Walk(tx, kv.StorageChangeSet, dbutils.EncodeBlockNumber(block), 0, func(blockN uint64, k, v []byte) (bool, error) {
+	if err := changeset.ForEach(tx, kv.StorageChangeSet, dbutils.EncodeBlockNumber(block), func(blockN uint64, k, v []byte) error {
 		if bytes.Equal(k, key) {
 			fmt.Printf("Found in block %d with value %x\n", blockN, v)
 		}
-		return true, nil
+		return nil
 	}); err != nil {
 		return err
 	}
@@ -2267,7 +2293,7 @@ func devTx(chaindata string) error {
 	tool.Check(err)
 	cc, err := rawdb.ReadChainConfig(tx, b.Hash())
 	tool.Check(err)
-	txn := types.NewTransaction(1, common.Address{}, uint256.NewInt(100), 100_000, uint256.NewInt(1), []byte{1})
+	txn := types.NewTransaction(2, common.Address{}, uint256.NewInt(100), 100_000, uint256.NewInt(1), []byte{1})
 	signedTx, err := types.SignTx(txn, *types.LatestSigner(cc), core.DevnetSignPrivateKey)
 	tool.Check(err)
 	buf := bytes.NewBuffer(nil)
@@ -2331,9 +2357,6 @@ func main() {
 	case "val-tx-lookup-2":
 		ValidateTxLookups2(*chaindata)
 
-	case "modiAccounts":
-		getModifiedAccounts(*chaindata)
-
 	case "slice":
 		dbSlice(*chaindata, *bucket, common.FromHex(*hash))
 
@@ -2404,7 +2427,7 @@ func main() {
 		err = snapSizes(*chaindata)
 
 	case "dumpState":
-		err = dumpState(*chaindata)
+		err = dumpState(*chaindata, uint64(*block))
 
 	case "readCallTraces":
 		err = readCallTraces(*chaindata, uint64(*block))

cmd/integration/commands/flags.go

@@ -25,6 +25,8 @@ var (
 	txtrace                        bool // Whether to trace the execution (should only be used together eith `block`)
 	pruneFlag                      string
 	pruneH, pruneR, pruneT, pruneC uint64
+	pruneHBefore, pruneRBefore     uint64
+	pruneTBefore, pruneCBefore     uint64
 	experiments                    []string
 	chain                          string // Which chain to use (mainnet, ropsten, rinkeby, goerli, etc.)
 )