Merge pull request #2190 from ledgerwatch/stable-2020-06-04

Author: mandrigin

Makefile

@@ -1,11 +1,11 @@
 GOBIN = $(CURDIR)/build/bin
-GOTEST = GODEBUG=cgocheck=2 go test ./... -p 1
+GOTEST = GODEBUG=cgocheck=0 go test ./... -p 2
 
 GIT_COMMIT ?= $(shell git rev-list -1 HEAD)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 GIT_TAG    ?= $(shell git describe --tags)
-GOBUILD = env GO111MODULE=on go build -trimpath -ldflags "-X main.gitCommit=${GIT_COMMIT} -X main.gitBranch=${GIT_BRANCH} -X main.gitTag=${GIT_TAG}"
-GO_DBG_BUILD = env CGO_CFLAGS='-O0 -g -DMDBX_BUILD_FLAGS_CONFIG="config.h"' GODEBUG=cgocheck=2 go build -trimpath -tags=debug -ldflags "-X main.gitCommit=${GIT_COMMIT} -X main.gitBranch=${GIT_BRANCH} -X main.gitTag=${GIT_TAG}" -gcflags=all="-N -l"  # see delve docs
+GOBUILD = env GO111MODULE=on go build -trimpath -ldflags "-X github.com/ledgerwatch/erigon/params.GitCommit=${GIT_COMMIT} -X github.com/ledgerwatch/erigon/params.GitBranch=${GIT_BRANCH} -X github.com/ledgerwatch/params.GitTag=${GIT_TAG}"
+GO_DBG_BUILD = env CGO_CFLAGS='-O0 -g -DMDBX_BUILD_FLAGS_CONFIG="config.h"' GODEBUG=cgocheck=2 go build -trimpath -tags=debug -ldflags "-X github.com/ledgerwatch/erigon/params.GitCommit=${GIT_COMMIT} -X github.com/ledgerwatch/erigon/params.GitBranch=${GIT_BRANCH} -X github.com/ledgerwatch/erigon/params.GitTag=${GIT_TAG}" -gcflags=all="-N -l"  # see delve docs
 
 GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
 GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
@@ -110,27 +110,22 @@ tracker:
 
 db-tools: mdbx
 	@echo "Building bb-tools"
-	go mod vendor; cd vendor/github.com/ledgerwatch/lmdb-go/dist; make clean mdb_stat mdb_copy mdb_dump mdb_drop mdb_load; cp mdb_stat $(GOBIN); cp mdb_copy $(GOBIN); cp mdb_dump $(GOBIN); cp mdb_drop $(GOBIN); cp mdb_load $(GOBIN); cd ../../../../..; rm -rf vendor
 
-	cd ethdb/mdbx/dist/ && make tools
+	cd ethdb/mdbx/dist/ && MDBX_BUILD_TIMESTAMP=unknown make tools
 	cp ethdb/mdbx/dist/mdbx_chk $(GOBIN)
 	cp ethdb/mdbx/dist/mdbx_copy $(GOBIN)
 	cp ethdb/mdbx/dist/mdbx_dump $(GOBIN)
 	cp ethdb/mdbx/dist/mdbx_drop $(GOBIN)
 	cp ethdb/mdbx/dist/mdbx_load $(GOBIN)
 	cp ethdb/mdbx/dist/mdbx_stat $(GOBIN)
-	@echo "Run \"$(GOBIN)/lmdb_stat -h\" to get info about lmdb file."
+	@echo "Run \"$(GOBIN)/mdbx_stat -h\" to get info about mdbx db file."
 
 mdbx:
 	@echo "Building mdbx"
 	@cd ethdb/mdbx/dist/ \
-		&& make clean && make config.h \
+		&& make clean && MDBX_BUILD_TIMESTAMP=unknown make config.h \
 		&& echo '#define MDBX_DEBUG 0' >> config.h \
 		&& echo '#define MDBX_FORCE_ASSERTIONS 0' >> config.h \
-		&& mv config.h config2.h \
-		&& tail -n +2 config2.h > config.h  \
-		&& rm -f config2.h \
-		&& cat config.h \
         && CFLAGS_EXTRA="-Wno-deprecated-declarations" make mdbx-static.o
 
 mdbx-dbg:
@@ -142,14 +137,7 @@ mdbx-dbg:
         && CFLAGS_EXTRA="-Wno-deprecated-declarations" CFLAGS='-O0 -g -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-error=attributes' make mdbx-static.o
 
 test: mdbx
-	TEST_DB=mdbx $(GOTEST) --timeout 15m
-
-test-lmdb:
-	TEST_DB=lmdb $(GOTEST)
-
-
-test-mdbx:
-	TEST_DB=mdbx $(GOTEST) --timeout 20m
+	$(GOTEST) --timeout 30m
 
 lint:
 	@./build/bin/golangci-lint run --config ./.golangci.yml
@@ -194,7 +182,7 @@ grpc:
 	rm -rf ./build/include*
 
 	$(eval PROTOC_TMP := $(shell mktemp -d))
-	cd $(PROTOC_TMP); curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v3.17.0/protoc-3.17.0-$(PROTOC_OS)-$(ARCH).zip -o protoc.zip
+	cd $(PROTOC_TMP); curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protoc-3.17.3-$(PROTOC_OS)-$(ARCH).zip -o protoc.zip
 	cd $(PROTOC_TMP); unzip protoc.zip && mv bin/protoc $(GOBIN) && mv include $(GOBIN)/..
 
 	$(GOBUILD) -o $(GOBIN)/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go # generates proto messages

README.md

@@ -25,7 +25,7 @@ Erigon is an implementation of Ethereum (aka "Ethereum client"), on the efficien
   + [Reporting security issues/concerns](#reporting-security-issues-concerns)
   + [Team](#team)
 - [Known issues](#known-issues)
-  + [`htop` shows incorrect memory usage on LMDB](#htop-shows-incorrect-memory-usage-when-using-lmdb)
+  + [`htop` shows incorrect memory usage](#htop-shows-incorrect-memory-usage)
 <!--te-->
 
 
@@ -107,9 +107,6 @@ Windows users may run erigon in 3 possible ways:
   security products for Windows and their respective detection algorythms and we understand this might make your experience with Windows builds uncomfortable. To
   workaround the issue you might either set exlusions for your antivirus specifically for `ethdb\mdbx\dist\CMakeFiles` folder or you can run erigon on Docker or WSL
 
-  Though is still possible to run erigon with **LMDB** database there's a caveat which might cause your experience with LMDB on Windows uncomfortable: data file allocation is fixed so you need to know in advance how much space you want to allocate for database file using the command line option `--lmdb.mapSize`.
-  Please be advised Erigon will completely remove LMDB support in future releases thus we warmly suggest to resync using the default MDBX database.
-
 * Use Docker :  see [docker-compose.yml](./docker-compose.yml)
 
 * Use WSL (Windows Subsystem for Linux) **strictly on version 2**. Under this option you can build Erigon just as you would on a regular Linux distribution. You can point your data also to any of the mounted Windows partitions (eg. `/mnt/c/[...]`, `/mnt/d/[...]` etc) but in such case be advised performance is impacted: this is due to the fact those mount points use `DrvFS` which is a [network file system](#blocks-execution-is-slow-on-cloud-network-drives) and, additionally, MDBX locks the db for exclusive access which implies only one process at a time can access data. This has consequences on the running of `rpcdaemon` which has to be configured as [Remote DB](#for-remote-db) even if it is executed on the very same computer.
@@ -284,9 +281,9 @@ Happy testing! 🥤
 Known issues
 ============
 
-### `htop` shows incorrect memory usage when using LMDB
+### `htop` shows incorrect memory usage
 
-Erigon's internal DB (LMDB) using `MemoryMap` - when OS does manage all `read, write, cache` operations instead of Application
+Erigon's internal DB (MDBX) using `MemoryMap` - when OS does manage all `read, write, cache` operations instead of Application
 ([linux](https://linux-kernel-labs.github.io/refs/heads/master/labs/memory_mapping.html), [windows](https://docs.microsoft.com/en-us/windows/win32/memory/file-mapping))
 
 `htop` on column `res` shows memory of "App + OS used to hold page cache for given App",

accounts/abi/bind/backends/simulated.go

@@ -66,7 +66,7 @@ var (
 type SimulatedBackend struct {
 	m         *stages.MockSentry
 	getHeader func(hash common.Hash, number uint64) *types.Header
-	checkTEVM func(hash common.Hash) (bool, error)
+	checkTEVM func(common.Hash) (bool, error)
 
 	mu              sync.Mutex
 	prependBlock    *types.Block

cmd/abigen/main.go

@@ -32,14 +32,11 @@ import (
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/internal/flags"
 	"github.com/ledgerwatch/erigon/log"
+	"github.com/ledgerwatch/erigon/params"
 	"github.com/urfave/cli"
 )
 
 var (
-	// Git SHA1 commit hash of the release (set via linker flags)
-	gitCommit = ""
-	gitDate   = ""
-
 	app *cli.App
 
 	// Flags needed by abigen
@@ -101,7 +98,7 @@ var (
 )
 
 func init() {
-	app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool")
+	app = flags.NewApp(params.GitCommit, "", "ethereum checkpoint helper tool")
 	app.Flags = []cli.Flag{
 		abiFlag,
 		binFlag,

cmd/cons/commands/root.go

@@ -14,7 +14,6 @@ import (
 var (
 	consensusAddr string // Address of the consensus engine <host>:<port>
 	datadir       string // Path to the working dir
-	database      string // Type of database (lmdb or mdbx)
 	config        string // `file:<path>`` to specify config file in file system, `embed:<path>`` to use embedded file, `test` to register test interface and receive config from test driver
 )
 
@@ -52,7 +51,6 @@ func must(err error) {
 func withDatadir(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&datadir, "datadir", paths.DefaultDataDir(), "directory where databases and temporary files are kept")
 	must(cmd.MarkFlagDirname("datadir"))
-	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 }
 
 func withApiAddr(cmd *cobra.Command) {
@@ -68,14 +66,6 @@ func openDatabase(path string) ethdb.RwKV {
 }
 
 func openKV(path string, exclusive bool) ethdb.RwKV {
-	if database == "lmdb" {
-		opts := ethdb.NewLMDB().Path(path)
-		if exclusive {
-			opts = opts.Exclusive()
-		}
-		return opts.MustOpen()
-	}
-
 	opts := ethdb.NewMDBX().Path(path)
 	if exclusive {
 		opts = opts.Exclusive()

cmd/devp2p/crawl.go

@@ -19,6 +19,7 @@ package main
 import (
 	"time"
 
+	"github.com/ledgerwatch/erigon/common/debug"
 	"github.com/ledgerwatch/erigon/log"
 	"github.com/ledgerwatch/erigon/p2p/enode"
 )
@@ -103,6 +104,7 @@ loop:
 }
 
 func (c *crawler) runIterator(done chan<- enode.Iterator, it enode.Iterator) {
+	defer func() { debug.LogPanic(nil, true, recover()) }()
 	defer func() { done <- it }()
 	for it.Next() {
 		select {

cmd/devp2p/main.go

@@ -30,12 +30,10 @@ import (
 
 var (
 	// Git information set by linker when building with ci.go.
-	gitCommit string
-	gitDate   string
-	app       = &cli.App{
+	app = &cli.App{
 		Name:        filepath.Base(os.Args[0]),
 		Usage:       "go-ethereum devp2p tool",
-		Version:     params.VersionWithCommit(gitCommit, gitDate),
+		Version:     params.VersionWithCommit(params.GitCommit, ""),
 		Writer:      os.Stdout,
 		HideVersion: true,
 	}

cmd/erigon/main.go

@@ -6,20 +6,17 @@ import (
 	"os"
 
 	"github.com/ledgerwatch/erigon/cmd/utils"
+	"github.com/ledgerwatch/erigon/common/debug"
 	"github.com/ledgerwatch/erigon/log"
+	"github.com/ledgerwatch/erigon/params"
 	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
 	"github.com/ledgerwatch/erigon/turbo/node"
 	"github.com/urfave/cli"
 )
 
-var (
-	// Following vars are injected through the build flags (see Makefile)
-	gitCommit string
-	gitBranch string
-	gitTag    string
-)
-
 func main() {
+	// catch panics from main thread and logs stack trace into a file
+	defer func() { debug.LogPanic(nil, true, recover()) }()
 	// creating a erigon-api app with all defaults
 	app := erigoncli.MakeApp(runErigon, erigoncli.DefaultFlags)
 	if err := app.Run(os.Args); err != nil {
@@ -30,12 +27,11 @@ func main() {
 
 func runErigon(cliCtx *cli.Context) {
 	// creating staged sync with all default parameters
-
 	ctx, _ := utils.RootContext()
 
 	// initializing the node and providing the current git commit there
-	log.Info("Build info", "git_branch", gitBranch, "git_tag", gitTag, "git_commit", gitCommit)
-	eri := node.New(cliCtx, node.Params{GitCommit: gitCommit, GitBranch: gitBranch})
+	log.Info("Build info", "git_branch", params.GitBranch, "git_tag", params.GitTag, "git_commit", params.GitCommit)
+	eri := node.New(cliCtx, node.Params{})
 	eri.SetP2PListenFunc(func(network, addr string) (net.Listener, error) {
 		var lc net.ListenConfig
 		return lc.Listen(ctx, network, addr)

cmd/evm/internal/t8ntool/execution.go

@@ -52,7 +52,7 @@ type ExecutionResult struct {
 	LogsHash    common.Hash    `json:"logsHash"`
 	Bloom       types.Bloom    `json:"logsBloom"        gencodec:"required"`
 	Receipts    types.Receipts `json:"receipts"`
-	Rejected    []int          `json:"rejected,omitempty"`
+	Rejected    []*rejectedTx  `json:"rejected,omitempty"`
 }
 
 type ommer struct {
@@ -72,6 +72,11 @@ type stEnv struct {
 	BaseFee     *big.Int                            `json:"currentBaseFee,omitempty"`
 }
 
+type rejectedTx struct {
+	Index int    `json:"index"`
+	Err   string `json:"error"`
+}
+
 type stEnvMarshaling struct {
 	Coinbase   common.UnprefixedAddress
 	Difficulty *math.HexOrDecimal256
@@ -106,7 +111,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		signer      = types.MakeSigner(chainConfig, pre.Env.Number)
 		gaspool     = new(core.GasPool)
 		blockHash   = common.Hash{0x13, 0x37}
-		rejectedTxs []int
+		rejectedTxs []*rejectedTx
 		includedTxs types.Transactions
 		gasUsed     = uint64(0)
 		receipts    = make(types.Receipts, 0)
@@ -143,8 +148,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	for i, tx := range txs {
 		msg, err := tx.AsMessage(*signer, pre.Env.BaseFee)
 		if err != nil {
-			log.Info("rejected tx", "index", i, "hash", tx.Hash(), "error", err)
-			rejectedTxs = append(rejectedTxs, i)
+			log.Warn("rejected tx", "index", i, "hash", tx.Hash(), "error", err)
+			rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
 			continue
 		}
 		tracer, err := getTracerFn(txIndex, tx.Hash())
@@ -163,7 +168,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		if err != nil {
 			ibs.RevertToSnapshot(snapshot)
 			log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From(), "error", err)
-			rejectedTxs = append(rejectedTxs, i)
+			rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
 			continue
 		}
 		includedTxs = append(includedTxs, tx)

cmd/evm/main.go

@@ -25,14 +25,12 @@ import (
 	"github.com/ledgerwatch/erigon/cmd/evm/internal/t8ntool"
 	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/internal/flags"
+	"github.com/ledgerwatch/erigon/params"
 	"github.com/urfave/cli"
 )
 
-var gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags)
-var gitDate = ""
-
 var (
-	app = flags.NewApp(gitCommit, gitDate, "the evm command line interface")
+	app = flags.NewApp(params.GitCommit, "", "the evm command line interface")
 
 	DebugFlag = cli.BoolFlag{
 		Name:  "debug",