master..merge/foundation release/1.10.23 tests generate fixes statefix difficulty test v2

.github/workflows/evmc.yml

@@ -8,8 +8,8 @@ on:
 
 jobs:
 
-  build-ewasm:
-    name: EWASM State Tests
+  build-evmc:
+    name: EVMC/EVM+EWASM State Tests
     runs-on: ubuntu-latest
     steps:
 
@@ -22,9 +22,31 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - name: Install cmake
+        run: |
+          sudo apt-get update -y
+          sudo apt-get upgrade -y
+          sudo apt-get install -y cmake
+          cmake --version
+
+      - name: Install necessary GLIBCXX version
+        run: |
+          strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep GLIBCXX
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
+          sudo apt-get update -y
+          sudo apt-get install -y gcc-9 g++-9
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          g++ --version
+          gcc --version     
+          strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep GLIBCXX
+
       - name: Get dependencies
         run: |
           go get -v -t -d ./...
+          git config --global --add safe.directory $(pwd)/tests/evm-benchmarks
+          git config --global --add safe.directory $(pwd)/tests/testdata
+          git config --global --add safe.directory $(pwd)/tests/testdata/LegacyTests
           git submodule update --init --recursive
           export GOBIN=${HOME}/go/bin
           mkdir -p "${GOBIN}"

.github/workflows/go-generate-check.yml

@@ -0,0 +1,67 @@
+name: Developer helper
+on: pull_request
+env:
+  # GENERATE_EXCEPTIONS are exceptions made to the 'go generate' command.
+  # These patterns are matched (negatively) against 'go list' output.
+  #
+  # - trezor: Ignore generating trezor package based on this comment:
+  #   https://github.com/ethereum/go-ethereum/blob/master/accounts/usbwallet/trezor/trezor.go#L21-L43
+  GENERATE_EXCEPTIONS: |
+    trezor
+jobs:
+  go-generate-check:
+    name: Check if "go generate" has been run
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        id: go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install deps
+        id: install-deps
+        run: |
+          SOLC_BIN=solc-linux-amd64-v0.6.0+commit.26b70077
+          curl -OL https://binaries.soliditylang.org/linux-amd64/$SOLC_BIN
+          sudo mv $SOLC_BIN /usr/bin/solc
+          sudo chmod +x /usr/bin/solc
+
+        shell: bash
+      - name: Install devtools
+        id: install-devtools
+        run: make devtools
+      - name: Run go:generate
+        id: go-generate
+        run: |
+          list="$(go list ./...)"
+          for pattern in ${GENERATE_EXCEPTIONS[@]}; do
+            list="$(grep -v "$pattern" <<< "$list")"
+          done
+          go generate "$list"
+      - name: Revert custom generated files modifications before comparing them
+        id: revert-custom-generated-modifications
+        run: |
+          # NOTE to developers checking what triggered this alert.
+          # This script is meant to alert you if some files have to be regenerated using `go generate`.
+          # If this happens, you have to run `go generate ./...` and then check the below commits that are being reverted and reapply them, after considering if they are needed.
+
+          git config user.name github-actions
+          git config user.email [email protected]
+
+          # Intentionally revert this commit which has a custom modification to the genesis unmarshaling,
+          # with regards reading different genesis formats origniating from different clients
+          # This way, this script can alert us on any code changes that have to be applied on if file gets changed.
+          git revert --no-edit 4b2cf83737ffe7c46c334a11414d151de049e0b3
+
+      - name: Check for modified files
+        id: git-check
+        run: |
+          if ! git diff-index --quiet HEAD --; then
+            echo "🔴 ERROR: There are modified files after running 'go generate'"
+            git status
+            exit 1
+          fi

.gitmodules

@@ -1,8 +1,12 @@
 [submodule "tests"]
 	path = tests/testdata
-	url = https://github.com/etclabscore/tests
+	url = https://github.com/ethereum/tests
 	shallow = true
 [submodule "evm-benchmarks"]
 	path = tests/evm-benchmarks
 	url = https://github.com/ipsilon/evm-benchmarks
 	shallow = true
+[submodule "tests-etc"]
+	path = tests/testdata-etc
+	url = https://github.com/etclabscore/tests
+	shallow = true

Dockerfile

@@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/
 RUN cd /go-ethereum && go mod download
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth
+RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM alpine:latest

Dockerfile.alltools

@@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/
 RUN cd /go-ethereum && go mod download
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && go run build/ci.go install
+RUN cd /go-ethereum && go run build/ci.go install -static
 
 # Pull all binaries into a second stage deploy alpine container
 FROM alpine:latest

Makefile

@@ -30,7 +30,7 @@ ios:
 	@echo "Import \"$(GOBIN)/Geth.framework\" to use the library."
 
 test:
-	$(GORUN) build/ci.go test
+	$(GORUN) build/ci.go test -timeout 20m
 
 # DEPRECATED.
 # No attempt will be made after the Istanbul fork to maintain
@@ -48,24 +48,16 @@ test-coregeth: \
 hera:
 	./build/hera.sh
 
-ssvm:
-	./build/ssvm.sh
-
 evmone:
 	./build/evmone.sh
 
-aleth-interpreter:
-	./build/aleth-interpreter.sh
-
 # Test EVMC support against various external interpreters.
-test-evmc: hera ssvm evmone aleth-interpreter
+test-evmc: hera evmone
 	go test -count 1 ./tests -run TestState -evmc.ewasm=$(ROOT_DIR)/build/_workspace/hera/build/src/libhera.so
-	go test -count 1 ./tests -run TestState -evmc.ewasm=$(ROOT_DIR)/build/_workspace/SSVM/build/tools/ssvm-evmc/libssvmEVMC.so
 	go test -count 1 ./tests -run TestState -evmc.evm=$(ROOT_DIR)/build/_workspace/evmone/lib/libevmone.so
-	go test -count 1 ./tests -run TestState -evmc.evm=$(ROOT_DIR)/build/_workspace/aleth/lib/libaleth-interpreter.so
 
 clean-evmc:
-	rm -rf ./build/_workspace/hera ./build/_workspace/SSVM ./build/_workspace/evmone ./build/_workspace/aleth
+	rm -rf ./build/_workspace/hera ./build/_workspace/evmone
 
 test-coregeth-features: \
 	test-coregeth-features-coregeth \
@@ -102,15 +94,19 @@ tests-generate-state: ## Generate state tests.
 	env COREGETH_TESTS_GENERATE_STATE_TESTS=on \
 	env COREGETH_TESTS_CHAINCONFIG_FEATURE_EQUIVALENCE_COREGETH=on \
 	go test -p 1 -v -timeout 60m ./tests -run TestGenStateAll
+	rm -rf ./tests/testdata-etc/GeneralStateTests
+	mv ./tests/testdata_generated/GeneralStateTests ./tests/testdata-etc/GeneralStateTests
+	rm -rf ./tests/testdata-etc/LegacyTests
+	mv ./tests/testdata_generated/LegacyTests ./tests/testdata-etc/LegacyTests
+	rm -rf ./tests/testdata_generated
 
 tests-generate-difficulty: ## Generate difficulty tests.
-	@echo "Generating difficulty tests configs."
-	env COREGETH_TESTS_GENERATE_DIFFICULTY_TESTS_CONFIGS=on \
-	go run build/ci.go test -v -timeout 10m ./tests -run TestDifficultyTestConfigGen
-
 	@echo "Generating difficulty tests."
 	env COREGETH_TESTS_GENERATE_DIFFICULTY_TESTS=on \
 	go run build/ci.go test -v -timeout 10m ./tests -run TestDifficultyGen
+	rm -rf ./tests/testdata-etc/DifficultyTests
+	mv ./tests/testdata_generated/DifficultyTests ./tests/testdata-etc/DifficultyTests
+	rm -rf ./tests/testdata_generated
 
 lint: ## Run linters.
 	$(GORUN) build/ci.go lint

accounts/abi/abi.go

@@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) {
 		args = event.Inputs
 	}
 	if args == nil {
-		return nil, errors.New("abi: could not locate named method or event")
+		return nil, fmt.Errorf("abi: could not locate named method or event: %s", name)
 	}
 	return args, nil
 }

accounts/abi/bind/backends/simulated.go

@@ -71,7 +71,8 @@ type SimulatedBackend struct {
 	pendingState    *state.StateDB // Currently pending state that will be the active on request
 	pendingReceipts types.Receipts // Currently receipts for the pending block
 
-	events *filters.EventSystem // Event system for filtering log events live
+	events       *filters.EventSystem  // for filtering log events live
+	filterSystem *filters.FilterSystem // for filtering database logs
 
 	config ctypes.ChainConfigurator
 }
@@ -89,7 +90,11 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc genesisT.Gen
 		blockchain: blockchain,
 		config:     genesis.Config,
 	}
-	backend.events = filters.NewEventSystem(&filterBackend{database, blockchain, backend}, false)
+
+	filterBackend := &filterBackend{database, blockchain, backend}
+	backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{})
+	backend.events = filters.NewEventSystem(backend.filterSystem, false)
+
 	backend.rollback(blockchain.CurrentBlock())
 	return backend
 }
@@ -611,7 +616,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 			// User specified the legacy gas field, convert to 1559 gas typing
 			call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice
 		} else {
-			// User specified 1559 gas feilds (or none), use those
+			// User specified 1559 gas fields (or none), use those
 			if call.GasFeeCap == nil {
 				call.GasFeeCap = new(big.Int)
 			}
@@ -691,7 +696,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter
 	var filter *filters.Filter
 	if query.BlockHash != nil {
 		// Block filter requested, construct a single-shot filter
-		filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain, b}, *query.BlockHash, query.Addresses, query.Topics)
+		filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics)
 	} else {
 		// Initialize unset filter boundaries to run from genesis to chain head
 		from := int64(0)
@@ -703,7 +708,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter
 			to = query.ToBlock.Int64()
 		}
 		// Construct the range filter
-		filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics)
+		filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics)
 	}
 	// Run the filter and return all the logs
 	logs, err := filter.Logs(ctx)
@@ -856,7 +861,8 @@ type filterBackend struct {
 	backend *SimulatedBackend
 }
 
-func (fb *filterBackend) ChainDb() ethdb.Database  { return fb.db }
+func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db }
+
 func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") }
 
 func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) {
@@ -882,19 +888,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ
 	return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil
 }
 
-func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
-	number := rawdb.ReadHeaderNumber(fb.db, hash)
-	if number == nil {
-		return nil, nil
-	}
-	receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config())
-	if receipts == nil {
-		return nil, nil
-	}
-	logs := make([][]*types.Log, len(receipts))
-	for i, receipt := range receipts {
-		logs[i] = receipt.Logs
-	}
+func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) {
+	logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config())
 	return logs, nil
 }
 

accounts/abi/reflect.go

@@ -99,7 +99,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 func set(dst, src reflect.Value) error {
 	dstType, srcType := dst.Type(), src.Type()
 	switch {
-	case dstType.Kind() == reflect.Interface && dst.Elem().IsValid():
+	case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()):
 		return set(dst.Elem(), src)
 	case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}):
 		return set(dst.Elem(), src)

accounts/abi/reflect_test.go

@@ -32,7 +32,7 @@ type reflectTest struct {
 
 var reflectTests = []reflectTest{
 	{
-		name: "OneToOneCorrespondance",
+		name: "OneToOneCorrespondence",
 		args: []string{"fieldA"},
 		struc: struct {
 			FieldA int `abi:"fieldA"`

accounts/abi/unpack_test.go

@@ -352,6 +352,11 @@ func TestMethodMultiReturn(t *testing.T) {
 		&[]interface{}{&expected.Int, &expected.String},
 		"",
 		"Can unpack into a slice",
+	}, {
+		&[]interface{}{&bigint, ""},
+		&[]interface{}{&expected.Int, expected.String},
+		"",
+		"Can unpack into a slice without indirection",
 	}, {
 		&[2]interface{}{&bigint, new(string)},
 		&[2]interface{}{&expected.Int, &expected.String},

accounts/keystore/account_cache_test.go

@@ -318,7 +318,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
 func TestUpdatedKeyfileContents(t *testing.T) {
 	t.Parallel()
 
-	// Create a temporary kesytore to test with
+	// Create a temporary keystore to test with
 	rand.Seed(time.Now().UnixNano())
 	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int()))
 	ks := NewKeyStore(dir, LightScryptN, LightScryptP)

accounts/keystore/file_cache.go

@@ -39,7 +39,7 @@ type fileCache struct {
 func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) {
 	t0 := time.Now()
 
-	// List all the failes from the keystore folder
+	// List all the files from the keystore folder
 	files, err := os.ReadDir(keyDir)
 	if err != nil {
 		return nil, nil, nil, err
@@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 			log.Trace("Ignoring file on account scan", "path", path)
 			continue
 		}
-		// Gather the set of all and fresly modified files
+		// Gather the set of all and freshly modified files
 		all.Add(path)
 
 		info, err := fi.Info()

accounts/keystore/keystore_test.go

@@ -214,7 +214,7 @@ func TestSignRace(t *testing.T) {
 // Tests that the wallet notifier loop starts and stops correctly based on the
 // addition and removal of wallet event subscriptions.
 func TestWalletNotifierLifecycle(t *testing.T) {
-	// Create a temporary kesytore to test with
+	// Create a temporary keystore to test with
 	_, ks := tmpKeyStore(t, false)
 
 	// Ensure that the notification updater is not running yet

accounts/usbwallet/trezor.go

@@ -196,10 +196,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er
 	if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil {
 		return common.Address{}, err
 	}
-	if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats
+	if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats
 		return common.BytesToAddress(addr), nil
 	}
-	if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats
+	if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats
 		return common.HexToAddress(addr), nil
 	}
 	return common.Address{}, errors.New("missing derived address")

accounts/usbwallet/wallet.go

@@ -380,7 +380,7 @@ func (w *wallet) selfDerive() {
 					// of legacy-ledger, the first account on the legacy-path will
 					// be shown to the user, even if we don't actively track it
 					if i < len(nextAddrs)-1 {
-						w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(<url>,<path>, false) to track",
+						w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(<url>,<path>, false) to track",
 							"path", path, "address", nextAddrs[i])
 						break
 					}