diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index d8148cfc97..3d5f978eba 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -10,10 +10,54 @@ jobs: validate-pr: runs-on: ubuntu-latest steps: + - name: Check for Spam PR + uses: actions/github-script@v7 + with: + script: | + const prTitle = context.payload.pull_request.title; + const spamRegex = /^(feat|chore|fix)(\(.*\))?\s*:/i; + + if (spamRegex.test(prTitle)) { + // Leave a comment explaining why + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: `## PR Closed as Spam + + This PR was automatically closed because the title format \`feat:\`, \`fix:\`, or \`chore:\` is commonly associated with spam contributions. + + If this is a legitimate contribution, please: + 1. Review our contribution guidelines + 2. Use the correct PR title format: \`directory, ...: description\` + 3. Open a new PR with the proper title format + + Thank you for your understanding.` + }); + + // Close the PR + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + state: 'closed' + }); + + core.setFailed('PR closed as spam due to suspicious title format'); + return; + } + + console.log('✅ PR passed spam check'); + + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check PR Title Format uses: actions/github-script@v7 with: script: | + const fs = require('fs'); + const path = require('path'); const prTitle = context.payload.pull_request.title; const titleRegex = /^([\w\s,{}/.]+): .+/; @@ -21,6 +65,22 @@ jobs: core.setFailed(`PR title "${prTitle}" does not match required format: directory, ...: description`); return; } + + const match = prTitle.match(titleRegex); + const dirPart = match[1]; + const directories = dirPart.split(',').map(d => d.trim()); + const missingDirs = []; + for (const dir of directories) { + const fullPath = path.join(process.env.GITHUB_WORKSPACE, dir); + if (!fs.existsSync(fullPath)) { + missingDirs.push(dir); + } + } + + if (missingDirs.length > 0) { + core.setFailed(`The following directories in the PR title do not exist: ${missingDirs.join(', ')}`); + return; + } console.log('✅ PR title format is valid'); diff --git a/SECURITY.md b/SECURITY.md index 0b497b44ae..d497248de5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -21,7 +21,6 @@ Audit reports are published in the `docs` folder: https://github.com/ethereum/go To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publicly disclosed security vulnerabilities. -Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number. The following key may be used to communicate sensitive information to developers. diff --git a/accounts/abi/bind/v2/dep_tree_test.go b/accounts/abi/bind/v2/dep_tree_test.go index e686e3fec4..b2470d8a16 100644 --- a/accounts/abi/bind/v2/dep_tree_test.go +++ b/accounts/abi/bind/v2/dep_tree_test.go @@ -158,10 +158,10 @@ func testLinkCase(tcInput linkTestCaseInput) error { overrideAddrs = make(map[rune]common.Address) ) // generate deterministic addresses for the override set. - rand.Seed(42) + rng := rand.New(rand.NewSource(42)) for contract := range tcInput.overrides { var addr common.Address - rand.Read(addr[:]) + rng.Read(addr[:]) overrideAddrs[contract] = addr overridesAddrs[addr] = struct{}{} } diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 3e4266924f..29c4bdf2ca 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -418,6 +418,7 @@ func (ks *KeyStore) Export(a accounts.Account, passphrase, newPassphrase string) if err != nil { return nil, err } + defer zeroKey(key.PrivateKey) var N, P int if store, ok := ks.storage.(*keyStorePassphrase); ok { N, P = store.scryptN, store.scryptP @@ -477,6 +478,7 @@ func (ks *KeyStore) Update(a accounts.Account, passphrase, newPassphrase string) if err != nil { return err } + defer zeroKey(key.PrivateKey) return ks.storage.StoreKey(a.URL.Path, key, newPassphrase) } diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index 0664dc2cdd..6311e8d90a 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -81,6 +81,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error */ passBytes := []byte(password) derivedKey := pbkdf2.Key(passBytes, passBytes, 2000, 16, sha256.New) + if len(cipherText)%aes.BlockSize != 0 { + return nil, errors.New("ciphertext must be a multiple of block size") + } plainText, err := aesCBCDecrypt(derivedKey, cipherText, iv) if err != nil { return nil, err diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index b3a7be8df0..1e0230dc45 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -300,6 +300,10 @@ func (s *SecureChannelSession) decryptAPDU(data []byte) ([]byte, error) { return nil, err } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid ciphertext length: %d", len(data)) + } + ret := make([]byte, len(data)) crypter := cipher.NewCBCDecrypter(a, s.iv) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index 81457b7da2..6f8ac0d8d9 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -43,6 +43,14 @@ const refreshCycle = time.Second // trashing. const refreshThrottling = 500 * time.Millisecond +const ( + // deviceUsagePage identifies Ledger devices by HID usage page (0xffa0) on Windows and macOS. + // See: https://github.com/LedgerHQ/ledger-live/blob/05a2980e838955a11a1418da638ef8ac3df4fb74/libs/ledgerjs/packages/hw-transport-node-hid-noevents/src/TransportNodeHid.ts + deviceUsagePage = 0xffa0 + // deviceInterface identifies Ledger devices by USB interface number (0) on Linux. + deviceInterface = 0 +) + // Hub is a accounts.Backend that can find and handle generic USB hardware wallets. type Hub struct { scheme string // Protocol scheme prefixing account and wallet URLs. @@ -82,6 +90,7 @@ func NewLedgerHub() (*Hub, error) { 0x0005, /* Ledger Nano S Plus */ 0x0006, /* Ledger Nano FTS */ 0x0007, /* Ledger Flex */ + 0x0008, /* Ledger Nano Gen5 */ 0x0000, /* WebUSB Ledger Blue */ 0x1000, /* WebUSB Ledger Nano S */ @@ -89,7 +98,8 @@ func NewLedgerHub() (*Hub, error) { 0x5000, /* WebUSB Ledger Nano S Plus */ 0x6000, /* WebUSB Ledger Nano FTS */ 0x7000, /* WebUSB Ledger Flex */ - }, 0xffa0, 0, newLedgerDriver) + 0x8000, /* WebUSB Ledger Nano Gen5 */ + }, deviceUsagePage, deviceInterface, newLedgerDriver) } // NewTrezorHubWithHID creates a new hardware wallet manager for Trezor devices. diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 52595a1621..80e63f1864 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -184,7 +184,7 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash return nil, accounts.ErrWalletClosed } // Ensure the wallet is capable of signing the given transaction - if w.version[0] < 1 && w.version[1] < 5 { + if w.version[0] < 1 || (w.version[0] == 1 && w.version[1] < 5) { //lint:ignore ST1005 brand name displayed on the console return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2]) } diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 0fd0415a9e..f1597ca1a7 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -632,7 +632,7 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID // data is not supported for Ledger wallets, so this method will always return // an error. func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) { - return w.SignText(account, accounts.TextHash(text)) + return w.SignText(account, text) } // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given diff --git a/arbcrypto/keccak.go b/arbcrypto/keccak.go index 61f0f33c3e..4218b8771f 100644 --- a/arbcrypto/keccak.go +++ b/arbcrypto/keccak.go @@ -5,9 +5,9 @@ package arbcrypto import ( "hash" - "golang.org/x/crypto/sha3" + "github.com/ethereum/go-ethereum/crypto/keccak" ) func NewLegacyKeccak256() hash.Hash { - return sha3.NewLegacyKeccak256() + return keccak.NewLegacyKeccak256() } diff --git a/arbitrum/recreatestate.go b/arbitrum/recreatestate.go index 996a08bfa3..a729565502 100644 --- a/arbitrum/recreatestate.go +++ b/arbitrum/recreatestate.go @@ -80,7 +80,7 @@ func AdvanceStateByBlock(ctx context.Context, bc *core.BlockChain, state *state. if logFunc != nil { logFunc(block.Header(), true) } - result, err := bc.Processor().Process(block, state, vmConfig) + result, err := bc.Processor().Process(ctx, block, state, vmConfig) if err != nil { return nil, nil, nil, fmt.Errorf("failed recreating state for block %d : %w", blockToRecreate, err) } diff --git a/beacon/blsync/engineclient.go b/beacon/blsync/engineclient.go index f9821fc6f3..9fc6a18a57 100644 --- a/beacon/blsync/engineclient.go +++ b/beacon/blsync/engineclient.go @@ -101,7 +101,16 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) params = []any{execData} ) switch fork { - case "electra": + case "altair", "bellatrix": + method = "engine_newPayloadV1" + case "capella": + method = "engine_newPayloadV2" + case "deneb": + method = "engine_newPayloadV3" + parentBeaconRoot := event.BeaconHead.ParentRoot + blobHashes := collectBlobHashes(event.Block) + params = append(params, blobHashes, parentBeaconRoot) + default: // electra, fulu and above method = "engine_newPayloadV4" parentBeaconRoot := event.BeaconHead.ParentRoot blobHashes := collectBlobHashes(event.Block) @@ -110,15 +119,6 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) hexRequests[i] = hexutil.Bytes(event.ExecRequests[i]) } params = append(params, blobHashes, parentBeaconRoot, hexRequests) - case "deneb": - method = "engine_newPayloadV3" - parentBeaconRoot := event.BeaconHead.ParentRoot - blobHashes := collectBlobHashes(event.Block) - params = append(params, blobHashes, parentBeaconRoot) - case "capella": - method = "engine_newPayloadV2" - default: - method = "engine_newPayloadV1" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) @@ -145,12 +145,12 @@ func (ec *engineClient) callForkchoiceUpdated(fork string, event types.ChainHead var method string switch fork { - case "deneb", "electra": - method = "engine_forkchoiceUpdatedV3" + case "altair", "bellatrix": + method = "engine_forkchoiceUpdatedV1" case "capella": method = "engine_forkchoiceUpdatedV2" - default: - method = "engine_forkchoiceUpdatedV1" + default: // deneb, electra, fulu and above + method = "engine_forkchoiceUpdatedV3" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) diff --git a/beacon/engine/gen_ed.go b/beacon/engine/gen_ed.go index 0ae5a3b8f1..6893d64a16 100644 --- a/beacon/engine/gen_ed.go +++ b/beacon/engine/gen_ed.go @@ -17,24 +17,23 @@ var _ = (*executableDataMarshaling)(nil) // MarshalJSON marshals as JSON. func (e ExecutableData) MarshalJSON() ([]byte, error) { type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` } var enc ExecutableData enc.ParentHash = e.ParentHash @@ -59,31 +58,29 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { enc.Withdrawals = e.Withdrawals enc.BlobGasUsed = (*hexutil.Uint64)(e.BlobGasUsed) enc.ExcessBlobGas = (*hexutil.Uint64)(e.ExcessBlobGas) - enc.ExecutionWitness = e.ExecutionWitness return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (e *ExecutableData) UnmarshalJSON(input []byte) error { type ExecutableData struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random *common.Hash `json:"prevRandao" gencodec:"required"` - Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash *common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random *common.Hash `json:"prevRandao" gencodec:"required"` + Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash *common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` } var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { @@ -157,8 +154,5 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.ExcessBlobGas != nil { e.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) } - if dec.ExecutionWitness != nil { - e.ExecutionWitness = dec.ExecutionWitness - } return nil } diff --git a/beacon/engine/types.go b/beacon/engine/types.go index ddb276ab09..da9b6568f2 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -73,24 +73,23 @@ type payloadAttributesMarshaling struct { // ExecutableData is the data necessary to execute an EL payload. type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom []byte `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number uint64 `json:"blockNumber" gencodec:"required"` - GasLimit uint64 `json:"gasLimit" gencodec:"required"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Timestamp uint64 `json:"timestamp" gencodec:"required"` - ExtraData []byte `json:"extraData" gencodec:"required"` - BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions [][]byte `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *uint64 `json:"blobGasUsed"` - ExcessBlobGas *uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom []byte `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number uint64 `json:"blockNumber" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + ExtraData []byte `json:"extraData" gencodec:"required"` + BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions [][]byte `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *uint64 `json:"blobGasUsed"` + ExcessBlobGas *uint64 `json:"excessBlobGas"` } // JSON type overrides for executableData. @@ -316,8 +315,7 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H RequestsHash: requestsHash, } return types.NewBlockWithHeader(header). - WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}). - WithWitness(data.ExecutionWitness), + WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}), nil } @@ -325,24 +323,23 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H // fields from the given block. It assumes the given block is post-merge block. func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.BlobTxSidecar, requests [][]byte) *ExecutionPayloadEnvelope { data := &ExecutableData{ - BlockHash: block.Hash(), - ParentHash: block.ParentHash(), - FeeRecipient: block.Coinbase(), - StateRoot: block.Root(), - Number: block.NumberU64(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - BaseFeePerGas: block.BaseFee(), - Timestamp: block.Time(), - ReceiptsRoot: block.ReceiptHash(), - LogsBloom: block.Bloom().Bytes(), - Transactions: encodeTransactions(block.Transactions()), - Random: block.MixDigest(), - ExtraData: block.Extra(), - Withdrawals: block.Withdrawals(), - BlobGasUsed: block.BlobGasUsed(), - ExcessBlobGas: block.ExcessBlobGas(), - ExecutionWitness: block.ExecutionWitness(), + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + FeeRecipient: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + BaseFeePerGas: block.BaseFee(), + Timestamp: block.Time(), + ReceiptsRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + Random: block.MixDigest(), + ExtraData: block.Extra(), + Withdrawals: block.Withdrawals(), + BlobGasUsed: block.BlobGasUsed(), + ExcessBlobGas: block.ExcessBlobGas(), } // Add blobs. diff --git a/beacon/light/canonical.go b/beacon/light/canonical.go index b5371493b4..56622425b2 100644 --- a/beacon/light/canonical.go +++ b/beacon/light/canonical.go @@ -69,7 +69,10 @@ func newCanonicalStore[T any](db ethdb.Iteratee, keyPrefix []byte) (*canonicalSt // databaseKey returns the database key belonging to the given period. func (cs *canonicalStore[T]) databaseKey(period uint64) []byte { - return binary.BigEndian.AppendUint64(append([]byte{}, cs.keyPrefix...), period) + key := make([]byte, len(cs.keyPrefix)+8) + copy(key, cs.keyPrefix) + binary.BigEndian.PutUint64(key[len(cs.keyPrefix):], period) + return key } // add adds the given item to the database. It also ensures that the range remains diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go index 5e41258053..7189767d9c 100644 --- a/beacon/light/sync/head_sync.go +++ b/beacon/light/sync/head_sync.go @@ -105,6 +105,7 @@ func (s *HeadSync) Process(requester request.Requester, events []request.Event) delete(s.serverHeads, event.Server) delete(s.unvalidatedOptimistic, event.Server) delete(s.unvalidatedFinality, event.Server) + delete(s.reqFinalityEpoch, event.Server) } } } diff --git a/beacon/params/checkpoint_hoodi.hex b/beacon/params/checkpoint_hoodi.hex index 2885d7c996..7bac591f96 100644 --- a/beacon/params/checkpoint_hoodi.hex +++ b/beacon/params/checkpoint_hoodi.hex @@ -1 +1 @@ -0x1bbf958008172591b6cbdb3d8d52e26998258e83d4bdb9eec10969d84519a6bd \ No newline at end of file +0xbb7a7f3c40d8ea0b450f91587db65d0f1c079669277e01a0426c8911702a863a \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex index 417e69a24b..c2886cc564 100644 --- a/beacon/params/checkpoint_mainnet.hex +++ b/beacon/params/checkpoint_mainnet.hex @@ -1 +1 @@ -0x2fe39a39b6f7cbd549e0f74d259de6db486005a65bd3bd92840dd6ce21d6f4c8 \ No newline at end of file +0x2af778d703186526a1b6304b423f338f11556206f618643c3f7fa0d7b1ef5c9b \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex index 02faf72187..55842f8ac0 100644 --- a/beacon/params/checkpoint_sepolia.hex +++ b/beacon/params/checkpoint_sepolia.hex @@ -1 +1 @@ -0x86686b2b366e24134e0e3969a9c5f3759f92e5d2b04785b42e22cc7d468c2107 \ No newline at end of file +0x48a89c9ea7ba19de2931797974cf8722344ab231c0edada278b108ef74125478 \ No newline at end of file diff --git a/beacon/params/config.go b/beacon/params/config.go index b01b739e07..437aa53788 100644 --- a/beacon/params/config.go +++ b/beacon/params/config.go @@ -38,7 +38,7 @@ import ( // across signing different data structures. const syncCommitteeDomain = 7 -var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB"} +var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB", "ELECTRA", "FULU"} // ClientConfig contains beacon light client configuration. type ClientConfig struct { @@ -103,6 +103,9 @@ func (c *ChainConfig) LoadForks(file []byte) error { epochs["GENESIS"] = 0 for key, value := range config { + if value == nil { + continue + } if strings.HasSuffix(key, "_FORK_VERSION") { name := key[:len(key)-len("_FORK_VERSION")] switch version := value.(type) { diff --git a/beacon/params/config_test.go b/beacon/params/config_test.go index 41e120469b..0b569b604c 100644 --- a/beacon/params/config_test.go +++ b/beacon/params/config_test.go @@ -15,6 +15,9 @@ ALTAIR_FORK_EPOCH: 1 EIP7928_FORK_VERSION: 0xb0000038 EIP7928_FORK_EPOCH: 18446744073709551615 +EIP7XXX_FORK_VERSION: +EIP7XXX_FORK_EPOCH: + BLOB_SCHEDULE: [] ` c := &ChainConfig{} diff --git a/beacon/params/networks.go b/beacon/params/networks.go index b35db34fd6..5dcf08cc5d 100644 --- a/beacon/params/networks.go +++ b/beacon/params/networks.go @@ -40,36 +40,39 @@ var ( GenesisTime: 1606824023, Checkpoint: common.HexToHash(checkpointMainnet), }). - AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). - AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). - AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). - AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}). - AddFork("DENEB", 269568, []byte{4, 0, 0, 0}). - AddFork("ELECTRA", 364032, []byte{5, 0, 0, 0}) + AddFork("GENESIS", 0, common.FromHex("0x00000000")). + AddFork("ALTAIR", 74240, common.FromHex("0x01000000")). + AddFork("BELLATRIX", 144896, common.FromHex("0x02000000")). + AddFork("CAPELLA", 194048, common.FromHex("0x03000000")). + AddFork("DENEB", 269568, common.FromHex("0x04000000")). + AddFork("ELECTRA", 364032, common.FromHex("0x05000000")). + AddFork("FULU", 411392, common.FromHex("0x06000000")) SepoliaLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), GenesisTime: 1655733600, Checkpoint: common.HexToHash(checkpointSepolia), }). - AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). - AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). - AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). - AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}). - AddFork("DENEB", 132608, []byte{144, 0, 0, 115}). - AddFork("ELECTRA", 222464, []byte{144, 0, 0, 116}) + AddFork("GENESIS", 0, common.FromHex("0x90000069")). + AddFork("ALTAIR", 50, common.FromHex("0x90000070")). + AddFork("BELLATRIX", 100, common.FromHex("0x90000071")). + AddFork("CAPELLA", 56832, common.FromHex("0x90000072")). + AddFork("DENEB", 132608, common.FromHex("0x90000073")). + AddFork("ELECTRA", 222464, common.FromHex("0x90000074")). + AddFork("FULU", 272640, common.FromHex("0x90000075")) HoleskyLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"), GenesisTime: 1695902400, Checkpoint: common.HexToHash(checkpointHolesky), }). - AddFork("GENESIS", 0, []byte{1, 1, 112, 0}). - AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}). - AddFork("BELLATRIX", 0, []byte{3, 1, 112, 0}). - AddFork("CAPELLA", 256, []byte{4, 1, 112, 0}). - AddFork("DENEB", 29696, []byte{5, 1, 112, 0}). - AddFork("ELECTRA", 115968, []byte{6, 1, 112, 0}) + AddFork("GENESIS", 0, common.FromHex("0x01017000")). + AddFork("ALTAIR", 0, common.FromHex("0x02017000")). + AddFork("BELLATRIX", 0, common.FromHex("0x03017000")). + AddFork("CAPELLA", 256, common.FromHex("0x04017000")). + AddFork("DENEB", 29696, common.FromHex("0x05017000")). + AddFork("ELECTRA", 115968, common.FromHex("0x06017000")). + AddFork("FULU", 165120, common.FromHex("0x07017000")) HoodiLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f"), @@ -82,5 +85,5 @@ var ( AddFork("CAPELLA", 0, common.FromHex("0x40000910")). AddFork("DENEB", 0, common.FromHex("0x50000910")). AddFork("ELECTRA", 2048, common.FromHex("0x60000910")). - AddFork("FULU", 18446744073709551615, common.FromHex("0x70000910")) + AddFork("FULU", 50688, common.FromHex("0x70000910")) ) diff --git a/beacon/types/beacon_block.go b/beacon/types/beacon_block.go index a2e31d5abf..82a0814a9f 100644 --- a/beacon/types/beacon_block.go +++ b/beacon/types/beacon_block.go @@ -52,7 +52,7 @@ func BlockFromJSON(forkName string, data []byte) (*BeaconBlock, error) { obj = new(capella.BeaconBlock) case "deneb": obj = new(deneb.BeaconBlock) - case "electra": + case "electra", "fulu": obj = new(electra.BeaconBlock) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/beacon/types/exec_header.go b/beacon/types/exec_header.go index ae79b00841..dbf5c54b36 100644 --- a/beacon/types/exec_header.go +++ b/beacon/types/exec_header.go @@ -45,7 +45,7 @@ func ExecutionHeaderFromJSON(forkName string, data []byte) (*ExecutionHeader, er switch forkName { case "capella": obj = new(capella.ExecutionPayloadHeader) - case "deneb", "electra": // note: the payload type was not changed in electra + case "deneb", "electra", "fulu": // note: the payload type was not changed in electra/fulu obj = new(deneb.ExecutionPayloadHeader) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/build/ci.go b/build/ci.go index 59c948acb3..abb7c4997f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -107,6 +107,18 @@ var ( Tags: "ziren", Env: map[string]string{"GOMIPS": "softfloat", "CGO_ENABLED": "0"}, }, + { + Name: "wasm-js", + GOOS: "js", + GOARCH: "wasm", + Tags: "example", + }, + { + Name: "wasm-wasi", + GOOS: "wasip1", + GOARCH: "wasm", + Tags: "example", + }, { Name: "example", Tags: "example", @@ -331,6 +343,10 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( } ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } + // TODO(gballet): revisit after the input api has been defined + if runtime.GOARCH == "wasm" { + ld = append(ld, "-gcflags=all=-d=softfloat") + } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } @@ -446,9 +462,14 @@ func doCheckGenerate() { ) pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} + excludes := []string{"tests/testdata", "build/cache", ".git"} + for i := range excludes { + excludes[i] = filepath.FromSlash(excludes[i]) + } + for _, mod := range goModules { // Compute the origin hashes of all the files - hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + hashes, err := build.HashFolder(mod, excludes) if err != nil { log.Fatal("Error computing hashes", "err", err) } @@ -458,7 +479,7 @@ func doCheckGenerate() { c.Dir = mod build.MustRun(c) // Check if generate file hashes have changed - generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + generated, err := build.HashFolder(mod, excludes) if err != nil { log.Fatalf("Error re-computing hashes: %v", err) } diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 39ff5d83c6..0000000000 --- a/circle.yml +++ /dev/null @@ -1,32 +0,0 @@ -machine: - services: - - docker - -dependencies: - cache_directories: - - "~/.ethash" # Cache the ethash DAG generated by hive for consecutive builds - - "~/.docker" # Cache all docker images manually to avoid lengthy rebuilds - override: - # Restore all previously cached docker images - - mkdir -p ~/.docker - - for img in `ls ~/.docker`; do docker load -i ~/.docker/$img; done - - # Pull in and hive, restore cached ethash DAGs and do a dry run - - go get -u github.com/karalabe/hive - - (cd ~/.go_workspace/src/github.com/karalabe/hive && mkdir -p workspace/ethash/ ~/.ethash) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && cp -r ~/.ethash/. workspace/ethash/) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=NONE --test=. --sim=. --loglevel=6) - - # Cache all the docker images and the ethash DAGs - - for img in `docker images | grep -v "^" | tail -n +2 | awk '{print $1}'`; do docker save $img > ~/.docker/`echo $img | tr '/' ':'`.tar; done - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/ethash/. ~/.ethash - -test: - override: - # Build Geth and move into a known folder - - make geth - - cp ./build/bin/geth $HOME/geth - - # Run hive and move all generated logs into the public artifacts folder - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=go-ethereum:local --override=$HOME/geth --test=. --sim=.) - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/logs/* $CIRCLE_ARTIFACTS diff --git a/cmd/devp2p/internal/ethtest/protocol.go b/cmd/devp2p/internal/ethtest/protocol.go index af76082318..a21d1ca7a1 100644 --- a/cmd/devp2p/internal/ethtest/protocol.go +++ b/cmd/devp2p/internal/ethtest/protocol.go @@ -86,9 +86,3 @@ func protoOffset(proto Proto) uint64 { panic("unhandled protocol") } } - -// msgTypePtr is the constraint for protocol message types. -type msgTypePtr[U any] interface { - *U - Kind() byte -} diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index f4fce0931f..7c1ca70cc0 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -937,10 +938,14 @@ func (s *Suite) snapGetTrieNodes(t *utesting.T, tc *trieNodesTest) error { } // write0 request + paths, err := rlp.EncodeToRawList(tc.paths) + if err != nil { + panic(err) + } req := &snap.GetTrieNodesPacket{ ID: uint64(rand.Int63()), Root: tc.root, - Paths: tc.paths, + Paths: paths, Bytes: tc.nBytes, } msg, err := conn.snapRequest(snap.GetTrieNodesMsg, req) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index c23360bf82..8bb488e358 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" "github.com/holiman/uint256" ) @@ -151,7 +152,11 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) { if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } - if !headersMatch(expected, headers.BlockHeadersRequest) { + received, err := headers.List.Items() + if err != nil { + t.Fatalf("invalid headers received: %v", err) + } + if !headersMatch(expected, received) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) } } @@ -237,7 +242,7 @@ concurrently, with different request IDs.`) // Wait for responses. // Note they can arrive in either order. - resp, err := collectResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { + resp, err := collectHeaderResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { if msg.RequestId != 111 && msg.RequestId != 222 { t.Fatalf("response with unknown request ID: %v", msg.RequestId) } @@ -248,17 +253,11 @@ concurrently, with different request IDs.`) } // Check if headers match. - resp1 := resp[111] - if expected, err := s.chain.GetHeaders(req1); err != nil { - t.Fatalf("failed to get expected headers for request 1: %v", err) - } else if !headersMatch(expected, resp1.BlockHeadersRequest) { - t.Fatalf("header mismatch for request ID %v: \nexpected %v \ngot %v", 111, expected, resp1) - } - resp2 := resp[222] - if expected, err := s.chain.GetHeaders(req2); err != nil { - t.Fatalf("failed to get expected headers for request 2: %v", err) - } else if !headersMatch(expected, resp2.BlockHeadersRequest) { - t.Fatalf("header mismatch for request ID %v: \nexpected %v \ngot %v", 222, expected, resp2) + if err := s.checkHeadersAgainstChain(req1, resp[111]); err != nil { + t.Fatal(err) + } + if err := s.checkHeadersAgainstChain(req2, resp[222]); err != nil { + t.Fatal(err) } } @@ -303,8 +302,8 @@ same request ID. The node should handle the request by responding to both reques // Wait for the responses. They can arrive in either order, and we can't tell them // apart by their request ID, so use the number of headers instead. - resp, err := collectResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { - id := uint64(len(msg.BlockHeadersRequest)) + resp, err := collectHeaderResponses(conn, 2, func(msg *eth.BlockHeadersPacket) uint64 { + id := uint64(msg.List.Len()) if id != 2 && id != 3 { t.Fatalf("invalid number of headers in response: %d", id) } @@ -315,26 +314,35 @@ same request ID. The node should handle the request by responding to both reques } // Check if headers match. - resp1 := resp[2] - if expected, err := s.chain.GetHeaders(request1); err != nil { - t.Fatalf("failed to get expected headers for request 1: %v", err) - } else if !headersMatch(expected, resp1.BlockHeadersRequest) { - t.Fatalf("headers mismatch: \nexpected %v \ngot %v", expected, resp1) - } - resp2 := resp[3] - if expected, err := s.chain.GetHeaders(request2); err != nil { - t.Fatalf("failed to get expected headers for request 2: %v", err) - } else if !headersMatch(expected, resp2.BlockHeadersRequest) { - t.Fatalf("headers mismatch: \nexpected %v \ngot %v", expected, resp2) + if err := s.checkHeadersAgainstChain(request1, resp[2]); err != nil { + t.Fatal(err) + } + if err := s.checkHeadersAgainstChain(request2, resp[3]); err != nil { + t.Fatal(err) + } +} + +func (s *Suite) checkHeadersAgainstChain(req *eth.GetBlockHeadersPacket, resp *eth.BlockHeadersPacket) error { + received2, err := resp.List.Items() + if err != nil { + return fmt.Errorf("invalid headers in response with request ID %v (%d items): %v", resp.RequestId, resp.List.Len(), err) + } + if expected, err := s.chain.GetHeaders(req); err != nil { + return fmt.Errorf("test chain failed to get expected headers for request: %v", err) + } else if !headersMatch(expected, received2) { + return fmt.Errorf("header mismatch for request ID %v (%d items): \nexpected %v \ngot %v", resp.RequestId, resp.List.Len(), expected, resp) } + return nil } // collectResponses waits for n messages of type T on the given connection. // The messsages are collected according to the 'identity' function. -func collectResponses[T any, P msgTypePtr[T]](conn *Conn, n int, identity func(P) uint64) (map[uint64]P, error) { - resp := make(map[uint64]P, n) +// +// This function is written in a generic way to handle +func collectHeaderResponses(conn *Conn, n int, identity func(*eth.BlockHeadersPacket) uint64) (map[uint64]*eth.BlockHeadersPacket, error) { + resp := make(map[uint64]*eth.BlockHeadersPacket, n) for range n { - r := new(T) + r := new(eth.BlockHeadersPacket) if err := conn.ReadMsg(ethProto, eth.BlockHeadersMsg, r); err != nil { return resp, fmt.Errorf("read error: %v", err) } @@ -373,10 +381,8 @@ and expects a response.`) if got, want := headers.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id") } - if expected, err := s.chain.GetHeaders(req); err != nil { - t.Fatalf("failed to get expected block headers: %v", err) - } else if !headersMatch(expected, headers.BlockHeadersRequest) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) + if err := s.checkHeadersAgainstChain(req, headers); err != nil { + t.Fatal(err) } } @@ -407,9 +413,8 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) { if got, want := resp.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in respond", got, want) } - bodies := resp.BlockBodiesResponse - if len(bodies) != len(req.GetBlockBodiesRequest) { - t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetBlockBodiesRequest), len(bodies)) + if resp.List.Len() != len(req.GetBlockBodiesRequest) { + t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetBlockBodiesRequest), resp.List.Len()) } } @@ -433,7 +438,7 @@ func (s *Suite) TestGetReceipts(t *utesting.T) { } } - // Create block bodies request. + // Create receipts request. req := ð.GetReceiptsPacket{ RequestId: 66, GetReceiptsRequest: (eth.GetReceiptsRequest)(hashes), @@ -449,8 +454,8 @@ func (s *Suite) TestGetReceipts(t *utesting.T) { if got, want := resp.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in respond", got, want) } - if len(resp.List) != len(req.GetReceiptsRequest) { - t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetReceiptsRequest), len(resp.List)) + if resp.List.Len() != len(req.GetReceiptsRequest) { + t.Fatalf("wrong receipts in response: expected %d receipts, got %d", len(req.GetReceiptsRequest), resp.List.Len()) } } @@ -804,7 +809,11 @@ on another peer connection using GetPooledTransactions.`) if got, want := msg.RequestId, req.RequestId; got != want { t.Fatalf("unexpected request id in response: got %d, want %d", got, want) } - for _, got := range msg.PooledTransactionsResponse { + responseTxs, err := msg.List.Items() + if err != nil { + t.Fatalf("invalid transactions in response: %v", err) + } + for _, got := range responseTxs { if _, exists := set[got.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", got.Hash()) } @@ -976,7 +985,9 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { if err := conn.ReadMsg(ethProto, eth.GetPooledTransactionsMsg, req); err != nil { t.Fatalf("reading pooled tx request failed: %v", err) } - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: test.resp} + + encTxs, _ := rlp.EncodeToRawList(test.resp) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { t.Fatalf("writing pooled tx response failed: %v", err) } @@ -1104,7 +1115,8 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types // the good peer is connected, and has announced the tx. // proceed to send the incorrect one from the bad peer. - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: eth.PooledTransactionsResponse(types.Transactions{badTx})} + encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{badTx}) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { errc <- fmt.Errorf("writing pooled tx response failed: %v", err) return @@ -1164,7 +1176,8 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types return } - resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, PooledTransactionsResponse: eth.PooledTransactionsResponse(types.Transactions{tx})} + encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{tx}) + resp := eth.PooledTransactionsPacket{RequestId: req.RequestId, List: encTxs} if err := conn.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil { errc <- fmt.Errorf("writing pooled tx response failed: %v", err) return diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index cbbbbce8d9..8ce26f3e1a 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/ethereum/go-ethereum/rlp" ) // sendTxs sends the given transactions to the node and @@ -51,7 +52,8 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { return fmt.Errorf("peering failed: %v", err) } - if err = sendConn.Write(ethProto, eth.TransactionsMsg, eth.TransactionsPacket(txs)); err != nil { + encTxs, _ := rlp.EncodeToRawList(txs) + if err = sendConn.Write(ethProto, eth.TransactionsMsg, eth.TransactionsPacket{RawList: encTxs}); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } @@ -68,7 +70,8 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { } switch msg := msg.(type) { case *eth.TransactionsPacket: - for _, tx := range *msg { + txs, _ := msg.Items() + for _, tx := range txs { got[tx.Hash()] = true } case *eth.NewPooledTransactionHashesPacket: @@ -80,9 +83,10 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error { if err != nil { t.Logf("invalid GetBlockHeaders request: %v", err) } + encHeaders, _ := rlp.EncodeToRawList(headers) recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ - RequestId: msg.RequestId, - BlockHeadersRequest: headers, + RequestId: msg.RequestId, + List: encHeaders, }) default: return fmt.Errorf("unexpected eth wire msg: %s", pretty.Sdump(msg)) @@ -167,9 +171,10 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error { if err != nil { t.Logf("invalid GetBlockHeaders request: %v", err) } + encHeaders, _ := rlp.EncodeToRawList(headers) recvConn.Write(ethProto, eth.BlockHeadersMsg, ð.BlockHeadersPacket{ - RequestId: msg.RequestId, - BlockHeadersRequest: headers, + RequestId: msg.RequestId, + List: encHeaders, }) default: return fmt.Errorf("unexpected eth message: %v", pretty.Sdump(msg)) diff --git a/cmd/era/main.go b/cmd/era/main.go index 35a889d4dc..1c26f44ad4 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/params" @@ -53,7 +55,7 @@ var ( eraSizeFlag = &cli.IntFlag{ Name: "size", Usage: "number of blocks per era", - Value: era.MaxEra1Size, + Value: era.MaxSize, } txsFlag = &cli.BoolFlag{ Name: "txs", @@ -131,7 +133,7 @@ func block(ctx *cli.Context) error { return nil } -// info prints some high-level information about the era1 file. +// info prints some high-level information about the era file. func info(ctx *cli.Context) error { epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) if err != nil { @@ -142,33 +144,34 @@ func info(ctx *cli.Context) error { return err } defer e.Close() - acc, err := e.Accumulator() - if err != nil { - return fmt.Errorf("error reading accumulator: %w", err) + var ( + accHex string + tdStr string + ) + if acc, err := e.Accumulator(); err == nil { + accHex = acc.Hex() } - td, err := e.InitialTD() - if err != nil { - return fmt.Errorf("error reading total difficulty: %w", err) + if td, err := e.InitialTD(); err == nil { + tdStr = td.String() } info := struct { - Accumulator common.Hash `json:"accumulator"` - TotalDifficulty *big.Int `json:"totalDifficulty"` - StartBlock uint64 `json:"startBlock"` - Count uint64 `json:"count"` + Accumulator string `json:"accumulator,omitempty"` + TotalDifficulty string `json:"totalDifficulty,omitempty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` }{ - acc, td, e.Start(), e.Count(), + accHex, tdStr, e.Start(), e.Count(), } b, _ := json.MarshalIndent(info, "", " ") fmt.Println(string(b)) return nil } -// open opens an era1 file at a certain epoch. -func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { - var ( - dir = ctx.String(dirFlag.Name) - network = ctx.String(networkFlag.Name) - ) +// open opens an era file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (era.Era, error) { + dir := ctx.String(dirFlag.Name) + network := ctx.String(networkFlag.Name) + entries, err := era.ReadDir(dir, network) if err != nil { return nil, fmt.Errorf("error reading era dir: %w", err) @@ -176,7 +179,28 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return era.Open(filepath.Join(dir, entries[epoch])) + path := filepath.Join(dir, entries[epoch]) + return openByPath(path) +} + +// openByPath tries to open a single file as either eraE or era1 based on extension, +// falling back to the other reader if needed. +func openByPath(path string) (era.Era, error) { + switch strings.ToLower(filepath.Ext(path)) { + case ".erae": + if e, err := execdb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + case ".era1": + if e, err := onedb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + } + return nil, fmt.Errorf("unsupported or unreadable era file: %s", path) } // verify checks each era1 file in a directory to ensure it is well-formed and @@ -203,18 +227,58 @@ func verify(ctx *cli.Context) error { return fmt.Errorf("error reading %s: %w", dir, err) } - if len(entries) != len(roots) { - return errors.New("number of era1 files should match the number of accumulator hashes") + // Build the verification list respecting the rule: + // era1: must have accumulator, always verify + // erae: verify only if accumulator exists (pre-merge) + + // Build list of files to verify. + verify := make([]string, 0, len(entries)) + + for _, name := range entries { + path := filepath.Join(dir, name) + ext := strings.ToLower(filepath.Ext(name)) + + switch ext { + case ".era1": + e, err := onedb.Open(path) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr != nil { + return fmt.Errorf("era1 file %s missing accumulator: %w", name, accErr) + } + verify = append(verify, path) + + case ".erae": + e, err := execdb.Open(path) + if err != nil { + return fmt.Errorf("error opening erae file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr == nil { + verify = append(verify, path) // pre-merge only + } + default: + return fmt.Errorf("unsupported era file: %s", name) + } + } + + if len(verify) != len(roots) { + return fmt.Errorf("mismatch between eras to verify (%d) and provided roots (%d)", len(verify), len(roots)) } // Verify each epoch matches the expected root. for i, want := range roots { // Wrap in function so defers don't stack. err := func() error { - name := entries[i] - e, err := era.Open(filepath.Join(dir, name)) + path := verify[i] + name := filepath.Base(path) + e, err := openByPath(path) if err != nil { - return fmt.Errorf("error opening era1 file %s: %w", name, err) + return fmt.Errorf("error opening era file %s: %w", name, err) } defer e.Close() // Read accumulator and check against expected. @@ -243,7 +307,7 @@ func verify(ctx *cli.Context) error { } // checkAccumulator verifies the accumulator matches the data in the Era. -func checkAccumulator(e *era.Era) error { +func checkAccumulator(e era.Era) error { var ( err error want common.Hash @@ -257,7 +321,7 @@ func checkAccumulator(e *era.Era) error { if td, err = e.InitialTD(); err != nil { return fmt.Errorf("error reading total difficulty: %w", err) } - it, err := era.NewIterator(e) + it, err := e.Iterator() if err != nil { return fmt.Errorf("error making era iterator: %w", err) } @@ -290,9 +354,13 @@ func checkAccumulator(e *era.Era) error { if rr != block.ReceiptHash() { return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) } - hashes = append(hashes, block.Hash()) - td.Add(td, block.Difficulty()) - tds = append(tds, new(big.Int).Set(td)) + // Only include pre-merge blocks in accumulator calculation. + // Post-merge blocks have difficulty == 0. + if block.Difficulty().Sign() > 0 { + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } } if it.Error() != nil { return fmt.Errorf("error reading block %d: %w", it.Number(), it.Error()) diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index f6538b1356..c6fac5396e 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -17,16 +17,18 @@ package main import ( + "bufio" "encoding/json" - "errors" "fmt" "maps" "os" "regexp" "slices" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" ) @@ -34,33 +36,52 @@ import ( var blockTestCommand = &cli.Command{ Action: blockTestCmd, Name: "blocktest", - Usage: "Executes the given blockchain tests", + Usage: "Executes the given blockchain tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ DumpFlag, HumanReadableFlag, RunFlag, WitnessCrossCheckFlag, + FuzzFlag, }, traceFlags), } func blockTestCmd(ctx *cli.Context) error { path := ctx.Args().First() - if len(path) == 0 { - return errors.New("path argument required") + + // If path is provided, run the tests at that path. + if len(path) != 0 { + var ( + collected = collectFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runBlockTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil } - var ( - collected = collectFiles(path) - results []testResult - ) - for _, fname := range collected { - r, err := runBlockTest(ctx, fname) + // Otherwise, read filenames from stdin and execute back-to-back. + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fname := scanner.Text() + if len(fname) == 0 { + return nil + } + results, err := runBlockTest(ctx, fname) if err != nil { return err } - results = append(results, r...) + // During fuzzing, we report the result after every block + if !ctx.IsSet(FuzzFlag.Name) { + report(ctx, results) + } } - report(ctx, results) return nil } @@ -79,6 +100,11 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { } tracer := tracerFromFlags(ctx) + // Suppress INFO logs during fuzzing + if ctx.IsSet(FuzzFlag.Name) { + log.SetDefault(log.NewLogger(log.DiscardHandler())) + } + // Pull out keys to sort and ensure tests are run in order. keys := slices.Sorted(maps.Keys(tests)) @@ -88,16 +114,35 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { if !re.MatchString(name) { continue } + test := tests[name] result := &testResult{Name: name, Pass: true} - if err := tests[name].Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { + var finalRoot *common.Hash + if err := test.Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { if s, _ := chain.State(); s != nil { result.State = dump(s) } } + // Capture final state root for end marker + if chain != nil { + root := chain.CurrentBlock().Root + finalRoot = &root + } }); err != nil { result.Pass, result.Error = false, err.Error() } + + // Always assign fork (regardless of pass/fail or tracer) + result.Fork = test.Network() + // Assign root if test succeeded + if result.Pass && finalRoot != nil { + result.Root = finalRoot + } + + // When fuzzing, write results after every block + if ctx.IsSet(FuzzFlag.Name) { + report(ctx, []testResult{*result}) + } results = append(results, *result) } return results, nil diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 17ca65a381..d8782cca9d 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -18,6 +18,7 @@ package t8ntool import ( "fmt" + stdmath "math" "math/big" "github.com/ethereum/go-ethereum/arbcrypto" @@ -43,8 +44,9 @@ import ( ) type Prestate struct { - Env stEnv `json:"env"` - Pre types.GenesisAlloc `json:"pre"` + Env stEnv `json:"env"` + Pre types.GenesisAlloc `json:"pre"` + TreeLeaves map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` } //go:generate go run github.com/fjl/gencodec -type ExecutionResult -field-override executionResultMarshaling -out gen_execresult.go @@ -145,7 +147,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return h } var ( - statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre) + isEIP4762 = chainConfig.IsVerkle(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) + statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp, params.MaxArbosVersionSupported) gaspool = new(core.GasPool) blockHash = common.Hash{0x13, 0x37} @@ -304,6 +307,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal) + + if isEIP4762 { + statedb.AccessEvents().AddAccount(w.Address, true, stdmath.MaxUint64) + } } // Gather the execution-layer triggered requests. @@ -365,8 +372,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, execRs.Requests = requests } - // Re-create statedb instance with new root upon the updated database - // for accessing latest states. + // Re-create statedb instance with new root for MPT mode statedb, err = state.New(root, statedb.Database()) if err != nil { return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) @@ -375,12 +381,17 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return statedb, execRs, body, nil } -func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { - tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool) *state.StateDB { + tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsVerkle: isBintrie}) sdb := state.NewDatabase(tdb, nil) - statedb, err := state.New(types.EmptyRootHash, sdb) + + root := types.EmptyRootHash + if isBintrie { + root = types.EmptyBinaryHash + } + statedb, err := state.New(root, sdb) if err != nil { - panic(fmt.Errorf("failed to create initial state: %v", err)) + panic(fmt.Errorf("failed to create initial statedb: %v", err)) } for addr, a := range accounts { statedb.SetCode(addr, a.Code, tracing.CodeChangeUnspecified) @@ -391,10 +402,15 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB } } // Commit and re-open to start with a clean state. - root, err := statedb.Commit(0, false, false) + root, err = statedb.Commit(0, false, false) if err != nil { panic(fmt.Errorf("failed to commit initial state: %v", err)) } + // If bintrie mode started, check if conversion happened + if isBintrie { + return statedb + } + // For MPT mode, reopen the state with the committed root statedb, err = state.New(root, sdb) if err != nil { panic(fmt.Errorf("failed to reopen state after commit: %v", err)) @@ -402,7 +418,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB return statedb } -func rlpHash(x interface{}) (h common.Hash) { +func rlpHash(x any) (h common.Hash) { hw := arbcrypto.NewLegacyKeccak256() rlp.Encode(hw, x) hw.Sum(h[:0]) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index f2606c86d1..a6ec33eacf 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -88,6 +88,14 @@ var ( "\t - into the file ", Value: "block.json", } + OutputBTFlag = &cli.StringFlag{ + Name: "output.vkt", + Usage: "Determines where to put the `BT` of the post-state.\n" + + "\t`stdout` - into the stdout output\n" + + "\t`stderr` - into the stderr output\n" + + "\t - into the file ", + Value: "vkt.json", + } InputAllocFlag = &cli.StringFlag{ Name: "input.alloc", Usage: "`stdin` or file name of where to find the prestate alloc to use.", @@ -123,6 +131,11 @@ var ( Usage: "`stdin` or file name of where to find the transactions list in RLP form.", Value: "txs.rlp", } + // TODO(@CPerezz): rename `Name` of the file in a follow-up PR (relays on EEST -> https://github.com/ethereum/execution-spec-tests/tree/verkle/main) + InputBTFlag = &cli.StringFlag{ + Name: "input.vkt", + Usage: "`stdin` or file name of where to find the prestate BT.", + } SealCliqueFlag = &cli.StringFlag{ Name: "seal.clique", Usage: "Seal block with Clique. `stdin` or file name of where to find the Clique sealing data.", diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 8b84a5626b..17f6577f23 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -115,9 +115,6 @@ func Transaction(ctx *cli.Context) error { } var results []result for it.Next() { - if err := it.Err(); err != nil { - return NewError(ErrorIO, err) - } var tx types.Transaction err := rlp.DecodeBytes(it.Value(), &tx) if err != nil { @@ -188,6 +185,10 @@ func Transaction(ctx *cli.Context) error { } results = append(results, r) } + if err := it.Err(); err != nil { + return NewError(ErrorIO, err) + } + out, err := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) return err diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 5b358be4ef..97de2241f5 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -28,15 +28,22 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/tests" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/holiman/uint256" "github.com/urfave/cli/v2" ) @@ -75,10 +82,11 @@ var ( ) type input struct { - Alloc types.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs []*txWithKey `json:"txs,omitempty"` - TxRlp string `json:"txsRlp,omitempty"` + Alloc types.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + BT map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Transition(ctx *cli.Context) error { @@ -90,16 +98,16 @@ func Transition(ctx *cli.Context) error { // stdin input or in files. // Check if anything needs to be read from stdin var ( - prestate Prestate - txIt txIterator // txs to apply - allocStr = ctx.String(InputAllocFlag.Name) - + prestate Prestate + txIt txIterator // txs to apply + allocStr = ctx.String(InputAllocFlag.Name) + btStr = ctx.String(InputBTFlag.Name) envStr = ctx.String(InputEnvFlag.Name) txStr = ctx.String(InputTxsFlag.Name) inputData = &input{} ) // Figure out the prestate alloc - if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { + if allocStr == stdinSelector || btStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) @@ -112,6 +120,13 @@ func Transition(ctx *cli.Context) error { } prestate.Pre = inputData.Alloc + if btStr != stdinSelector && btStr != "" { + if err := readFile(btStr, "BT", &inputData.BT); err != nil { + return err + } + } + prestate.TreeLeaves = inputData.BT + // Set the block environment if envStr != stdinSelector { var env stEnv @@ -182,9 +197,21 @@ func Transition(ctx *cli.Context) error { return err } // Dump the execution result - collector := make(Alloc) - s.DumpToCollector(collector, nil) - return dispatchOutput(ctx, baseDir, result, collector, body) + var ( + collector = make(Alloc) + btleaves map[common.Hash]hexutil.Bytes + ) + isBinary := chainConfig.IsVerkle(big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp) + if !isBinary { + s.DumpToCollector(collector, nil) + } else { + btleaves = make(map[common.Hash]hexutil.Bytes) + if err := s.DumpBinTrieLeaves(btleaves); err != nil { + return err + } + } + + return dispatchOutput(ctx, baseDir, result, collector, body, btleaves) } func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error { @@ -306,7 +333,7 @@ func saveFile(baseDir, filename string, data interface{}) error { // dispatchOutput writes the output data to either stderr or stdout, or to the specified // files -func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes) error { +func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes) error { stdOutObject := make(map[string]interface{}) stdErrObject := make(map[string]interface{}) dispatch := func(baseDir, fName, name string, obj interface{}) error { @@ -333,6 +360,13 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a if err := dispatch(baseDir, ctx.String(OutputBodyFlag.Name), "body", body); err != nil { return err } + // Only write bt output if we actually have binary trie leaves + if bt != nil { + if err := dispatch(baseDir, ctx.String(OutputBTFlag.Name), "vkt", bt); err != nil { + return err + } + } + if len(stdOutObject) > 0 { b, err := json.MarshalIndent(stdOutObject, "", " ") if err != nil { @@ -351,3 +385,168 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a } return nil } + +// BinKey computes the tree key given an address and an optional slot number. +func BinKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an optional slot number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + if ctx.Args().Len() == 2 { + slot, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding slot: %w", err) + } + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyStorageSlot(common.BytesToAddress(addr), slot)) + } else { + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyBasicData(common.BytesToAddress(addr))) + } + return nil +} + +// BinKeys computes a set of tree keys given a genesis alloc. +func BinKeys(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + // Figure out the prestate alloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + + collector := make(map[common.Hash]hexutil.Bytes) + it, err := bt.NodeIterator(nil) + if err != nil { + panic(err) + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + + output, err := json.MarshalIndent(collector, "", "") + if err != nil { + return fmt.Errorf("error outputting tree: %w", err) + } + + fmt.Println(string(output)) + + return nil +} + +// BinTrieRoot computes the root of a Binary Trie from a genesis alloc. +func BinTrieRoot(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + fmt.Println(bt.Hash().Hex()) + + return nil +} + +// TODO(@CPerezz): Should this go to `bintrie` module? +func genBinTrieFromAlloc(alloc core.GenesisAlloc, db database.NodeDatabase) (*bintrie.BinaryTrie, error) { + bt, err := bintrie.NewBinaryTrie(types.EmptyBinaryHash, db) + if err != nil { + return nil, err + } + for addr, acc := range alloc { + for slot, value := range acc.Storage { + err := bt.UpdateStorage(addr, slot.Bytes(), value.Big().Bytes()) + if err != nil { + return nil, fmt.Errorf("error inserting storage: %w", err) + } + } + account := &types.StateAccount{ + Balance: uint256.MustFromBig(acc.Balance), + Nonce: acc.Nonce, + CodeHash: crypto.Keccak256Hash(acc.Code).Bytes(), + Root: common.Hash{}, + } + err := bt.UpdateAccount(addr, account, len(acc.Code)) + if err != nil { + return nil, fmt.Errorf("error inserting account: %w", err) + } + err = bt.UpdateContractCode(addr, common.BytesToHash(account.CodeHash), acc.Code) + if err != nil { + return nil, fmt.Errorf("error inserting code: %w", err) + } + } + return bt, nil +} + +// BinaryCodeChunkKey computes the tree key of a code-chunk for a given address. +func BinaryCodeChunkKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an code-chunk number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + chunkNumberBytes, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding chunk number: %w", err) + } + var chunkNumber uint256.Int + chunkNumber.SetBytes(chunkNumberBytes) + + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyCodeChunk(common.BytesToAddress(addr), &chunkNumber)) + + return nil +} + +// BinaryCodeChunkCode returns the code chunkification for a given code. +func BinaryCodeChunkCode(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 1 { + return errors.New("invalid number of arguments: expecting a bytecode") + } + + bytecode, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + chunkedCode := bintrie.ChunkifyCode(bytecode) + fmt.Printf("%#x\n", chunkedCode) + + return nil +} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index bf5be9a359..57741b5f9c 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -55,6 +55,11 @@ var ( Usage: "benchmark the execution", Category: flags.VMCategory, } + FuzzFlag = &cli.BoolFlag{ + Name: "fuzz", + Usage: "adapts output format for fuzzing", + Category: flags.VMCategory, + } WitnessCrossCheckFlag = &cli.BoolFlag{ Name: "cross-check", Aliases: []string{"xc"}, @@ -146,16 +151,63 @@ var ( t8ntool.TraceEnableCallFramesFlag, t8ntool.OutputBasedir, t8ntool.OutputAllocFlag, + t8ntool.OutputBTFlag, t8ntool.OutputResultFlag, t8ntool.OutputBodyFlag, t8ntool.InputAllocFlag, t8ntool.InputEnvFlag, + t8ntool.InputBTFlag, t8ntool.InputTxsFlag, t8ntool.ForknameFlag, t8ntool.ChainIDFlag, t8ntool.RewardFlag, }, } + + verkleCommand = &cli.Command{ + Name: "verkle", + Aliases: []string{"vkt"}, + Usage: "Binary Trie helpers", + Subcommands: []*cli.Command{ + { + Name: "tree-keys", + Aliases: []string{"v"}, + Usage: "compute a set of binary trie keys, given their source addresses and optional slot numbers", + Action: t8ntool.BinKeys, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, + { + Name: "single-key", + Aliases: []string{"vk"}, + Usage: "compute the binary trie key given an address and optional slot number", + Action: t8ntool.BinKey, + }, + { + Name: "code-chunk-key", + Aliases: []string{"vck"}, + Usage: "compute the binary trie key given an address and chunk number", + Action: t8ntool.BinaryCodeChunkKey, + }, + { + Name: "chunkify-code", + Aliases: []string{"vcc"}, + Usage: "chunkify a given bytecode for a binary trie", + Action: t8ntool.BinaryCodeChunkCode, + }, + { + Name: "state-root", + Aliases: []string{"vsr"}, + Usage: "compute the state-root of a binary trie for the given alloc", + Action: t8ntool.BinTrieRoot, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, + }, + } + transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, @@ -210,6 +262,7 @@ func init() { stateTransitionCommand, transactionCommand, blockBuilderCommand, + verkleCommand, } app.Before = func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c5145bbfb7..1ccb78d622 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "regexp" @@ -43,6 +44,8 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/eradl" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -96,6 +99,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, utils.NoCompactionFlag, + utils.LogSlowBlockFlag, utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, utils.MetricsHTTPFlag, @@ -119,6 +123,8 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, }, utils.DatabaseFlags, debug.Flags), Before: func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) @@ -150,7 +156,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -161,7 +167,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: utils.DatabaseFlags, + Flags: slices.Concat([]cli.Flag{utils.EraFormatFlag}, utils.DatabaseFlags), Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -295,7 +301,7 @@ func initGenesis(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, stack, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle()) defer triedb.Close() - _, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides) + _, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides, nil) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } @@ -513,15 +519,27 @@ func importHistory(ctx *cli.Context) error { network = networks[0] } - if err := utils.ImportHistory(chain, dir, network); err != nil { + var ( + format = ctx.String(utils.EraFormatFlag.Name) + from func(era.ReadAtSeekCloser) (era.Era, error) + ) + switch format { + case "era1", "era": + from = onedb.From + case "erae": + from = execdb.From + default: + return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) + } + if err := utils.ImportHistory(chain, dir, network, from); err != nil { return err } + fmt.Printf("Import done in %v\n", time.Since(start)) return nil } -// exportHistory exports chain history in Era archives at a specified -// directory. +// exportHistory exports chain history in Era archives at a specified directory. func exportHistory(ctx *cli.Context) error { if ctx.Args().Len() != 3 { utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) @@ -547,10 +565,26 @@ func exportHistory(ctx *cli.Context) error { if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) } - err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) - if err != nil { + + var ( + format = ctx.String(utils.EraFormatFlag.Name) + filename func(network string, epoch int, root common.Hash) string + newBuilder func(w io.Writer) era.Builder + ) + switch format { + case "era1", "era": + newBuilder = func(w io.Writer) era.Builder { return onedb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return onedb.Filename(network, epoch, root) } + case "erae": + newBuilder = func(w io.Writer) era.Builder { return execdb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return execdb.Filename(network, epoch, root) } + default: + return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format) + } + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), newBuilder, filename); err != nil { utils.Fatalf("Export error: %v\n", err) } + fmt.Printf("Export done in %v\n", time.Since(start)) return nil } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index fcb315af97..7943d95d65 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -35,11 +35,11 @@ import ( "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/telemetry/tracesetup" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -240,9 +240,15 @@ func makeFullNode(ctx *cli.Context) *node.Node { cfg.Eth.OverrideVerkle = &v } - // Start metrics export if enabled + // Start metrics export if enabled. utils.SetupMetrics(&cfg.Metrics) + // Setup OpenTelemetry reporting if enabled. + if err := tracesetup.SetupTelemetry(cfg.Node.OpenTelemetry, stack); err != nil { + utils.Fatalf("failed to setup OpenTelemetry: %v", err) + } + + // Add Ethereum service. backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Create gauge with geth system and build information @@ -273,11 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node { // Configure synchronization override service var synctarget common.Hash if ctx.IsSet(utils.SyncTargetFlag.Name) { - hex := hexutil.MustDecode(ctx.String(utils.SyncTargetFlag.Name)) - if len(hex) != common.HashLength { - utils.Fatalf("invalid sync target length: have %d, want %d", len(hex), common.HashLength) + target := ctx.String(utils.SyncTargetFlag.Name) + if !common.IsHexHash(target) { + utils.Fatalf("sync target hash is not a valid hex hash: %s", target) } - synctarget = common.BytesToHash(hex) + synctarget = common.HexToHash(target) } utils.RegisterSyncOverrideService(stack, eth, synctarget, ctx.Bool(utils.ExitWhenSyncedFlag.Name)) @@ -399,9 +405,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + utils.Fatalf("Flags --%s, --%s, --%s are only available for influxdb-v2", utils.MetricsInfluxDBOrganizationFlag.Name, utils.MetricsInfluxDBTokenFlag.Name, utils.MetricsInfluxDBBucketFlag.Name) } else if enableExportV2 && v1FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + utils.Fatalf("Flags --%s, --%s are only available for influxdb-v1", utils.MetricsInfluxDBUsernameFlag.Name, utils.MetricsInfluxDBPasswordFlag.Name) } } } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 42d6b43ade..44895c8342 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -39,8 +39,9 @@ const ( // child g gets a temporary data directory. func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --holesky to make the 'writing genesis to disk' faster (no accounts) + // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--holesky", "--authrpc.port", "0", "--syncmode=full", "--port", "0", + allArgs := []string{"--holesky", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} return runGeth(t, append(allArgs, args...)...) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c57add0656..fb688793e3 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -41,7 +41,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" - "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" ) @@ -760,7 +759,7 @@ func showMetaData(ctx *cli.Context) error { data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } - table := tablewriter.NewWriter(os.Stdout) + table := rawdb.NewTableWriter(os.Stdout) table.SetHeader([]string{"Field", "Value"}) table.AppendBulk(data) table.Render() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 109b36836a..2291e0aafa 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -94,6 +94,8 @@ var ( utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, utils.LightKDFFlag, utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated @@ -118,6 +120,7 @@ var ( utils.MinerGasPriceFlag, utils.MinerEtherbaseFlag, // deprecated utils.MinerExtraDataFlag, + utils.MinerMaxBlobsFlag, utils.MinerRecommitIntervalFlag, utils.MinerPendingFeeRecipientFlag, utils.MinerNewPayloadTimeoutFlag, // deprecated @@ -156,6 +159,7 @@ var ( utils.BeaconGenesisTimeFlag, utils.BeaconCheckpointFlag, utils.BeaconCheckpointFileFlag, + utils.LogSlowBlockFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ @@ -191,6 +195,14 @@ var ( utils.BatchResponseMaxSize, utils.RPCTxSyncDefaultTimeoutFlag, utils.RPCTxSyncMaxTimeoutFlag, + utils.RPCGlobalRangeLimitFlag, + utils.RPCTelemetryFlag, + utils.RPCTelemetryEndpointFlag, + utils.RPCTelemetryUserFlag, + utils.RPCTelemetryPasswordFlag, + utils.RPCTelemetryInstanceIDFlag, + utils.RPCTelemetryTagsFlag, + utils.RPCTelemetrySampleRatioFlag, } metricsFlags = []cli.Flag{ @@ -239,7 +251,6 @@ func init() { javascriptCommand, // See misccmd.go: versionCommand, - versionCheckCommand, licenseCommand, // See config.go dumpConfigCommand, @@ -249,8 +260,6 @@ func init() { utils.ShowDeprecated, // See snapshot.go snapshotCommand, - // See verkle.go - verkleCommand, } if logTestCommand != nil { app.Commands = append(app.Commands, logTestCommand) diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index 2d31f3abe7..f5c0d55ebb 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -27,16 +27,6 @@ import ( ) var ( - VersionCheckUrlFlag = &cli.StringFlag{ - Name: "check.url", - Usage: "URL to use when checking vulnerabilities", - Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json", - } - VersionCheckVersionFlag = &cli.StringFlag{ - Name: "check.version", - Usage: "Version to check", - Value: version.ClientName(clientIdentifier), - } versionCommand = &cli.Command{ Action: printVersion, Name: "version", @@ -44,20 +34,6 @@ var ( ArgsUsage: " ", Description: ` The output of this command is supposed to be machine-readable. -`, - } - versionCheckCommand = &cli.Command{ - Action: versionCheck, - Flags: []cli.Flag{ - VersionCheckUrlFlag, - VersionCheckVersionFlag, - }, - Name: "version-check", - Usage: "Checks (online) for known Geth security vulnerabilities", - ArgsUsage: "", - Description: ` -The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, -and displays information about any security vulnerabilities that affect the currently executing version. `, } licenseCommand = &cli.Command{ diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index fe0bf97db8..62cc3111b5 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -643,11 +643,11 @@ func snapshotExportPreimages(ctx *cli.Context) error { var root common.Hash if ctx.NArg() > 1 { - rootBytes := common.FromHex(ctx.Args().Get(1)) - if len(rootBytes) != common.HashLength { + hash := ctx.Args().Get(1) + if !common.IsHexHash(hash) { return fmt.Errorf("invalid hash: %s", ctx.Args().Get(1)) } - root = common.BytesToHash(rootBytes) + root = common.HexToHash(hash) } else { headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { diff --git a/cmd/geth/testdata/vcheck/data.json b/cmd/geth/testdata/vcheck/data.json deleted file mode 100644 index e52fd84e67..0000000000 --- a/cmd/geth/testdata/vcheck/data.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig b/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig deleted file mode 100644 index 987ffe92bb..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RUQkliYstQBOKHklFEYCUjepz81dyUuDmIAxjAvXa+icjGuKcjtVfV06G7qfOMSpplS5EcntU12n+AnGNyuOM8zIctaIWcfG2w0= -trusted comment: timestamp:1752094689 file:data.json hashed -u2e4wo4HBTU6viQTSY/NVBHoWoPFJnnTvLZS0FYl3JdvSOYi6+qpbEsDhAIFqq/n8VmlS/fPqqf7vKCNiAgjAA== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 deleted file mode 100644 index 6b6aa900e3..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: timestamp:1752094703 file:data.json -cNyq3ZGlqo785HtWODb9ejWqF0HhSeXuLGXzC7z1IhnDrBObWBJngYd3qBG1dQcYlHQ+bgB/On5mSyMFn4UoCQ== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 deleted file mode 100644 index 704437de39..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: Here's a comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 deleted file mode 100644 index 806cd07316..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: One more (untrusted™) comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisign.pub b/cmd/geth/testdata/vcheck/minisign.pub deleted file mode 100644 index 183dce5f6b..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign public key 284E00B52C269624 -RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp diff --git a/cmd/geth/testdata/vcheck/minisign.sec b/cmd/geth/testdata/vcheck/minisign.sec deleted file mode 100644 index 5c50715b20..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign encrypted secret key -RWRTY0Iyz8kmPMKrqk6DCtlO9a33akKiaOQG1aLolqDxs52qvPoAAAACAAAAAAAAAEAAAAAArEiggdvyn6+WzTprirLtgiYQoU+ihz/HyGgjhuF+Pz2ddMduyCO+xjCHeq+vgVVW039fbsI8hW6LRGJZLBKV5/jdxCXAVVQE7qTQ6xpEdO0z8Z731/pV1hlspQXG2PNd16NMtwd9dWw= diff --git a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig b/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig deleted file mode 100644 index d704af7709..0000000000 --- a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: verify with signifykey.pub -RWSKLNhZb0KdARbMcGN40hbHzKQYZDgDOFhEUT1YpzMnqre/mbKJ8td/HVlG03Am1YCszATiI0DbnljjTy4iNHYwqBfzrFUqUg0= diff --git a/cmd/geth/testdata/vcheck/signifykey.pub b/cmd/geth/testdata/vcheck/signifykey.pub deleted file mode 100644 index 328f973ab4..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify public key -RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/ diff --git a/cmd/geth/testdata/vcheck/signifykey.sec b/cmd/geth/testdata/vcheck/signifykey.sec deleted file mode 100644 index 3279a2e58b..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify secret key -RWRCSwAAACpLQDLawSQCtI7eAVIvaiHzjTsTyJsfV5aKLNhZb0KdAWeICXJGa93/bHAcsY6jUh9I8RdEcDWEoGxmaXZC+IdVBPxDpkix9fBRGEUdKWHi3dOfqME0YRzErWI5AVg3cRw= diff --git a/cmd/geth/testdata/vcheck/vulnerabilities.json b/cmd/geth/testdata/vcheck/vulnerabilities.json deleted file mode 100644 index e52fd84e67..0000000000 --- a/cmd/geth/testdata/vcheck/vulnerabilities.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go deleted file mode 100644 index 67dc7257c0..0000000000 --- a/cmd/geth/verkle.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "os" - "slices" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-verkle" - "github.com/urfave/cli/v2" -) - -var ( - zero [32]byte - - verkleCommand = &cli.Command{ - Name: "verkle", - Usage: "A set of experimental verkle tree management commands", - Description: "", - Subcommands: []*cli.Command{ - { - Name: "verify", - Usage: "verify the conversion of a MPT into a verkle tree", - ArgsUsage: "", - Action: verifyVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle verify -This command takes a root commitment and attempts to rebuild the tree. - `, - }, - { - Name: "dump", - Usage: "Dump a verkle tree to a DOT file", - ArgsUsage: " [ ...]", - Action: expandVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle dump [ ...] -This command will produce a dot file representing the tree, rooted at . -in which key1, key2, ... are expanded. - `, - }, - }, - } -) - -// recurse into each child to ensure they can be loaded from the db. The tree isn't rebuilt -// (only its nodes are loaded) so there is no need to flush them, the garbage collector should -// take care of that for us. -func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error { - switch node := root.(type) { - case *verkle.InternalNode: - for i, child := range node.Children() { - childC := child.Commit().Bytes() - - childS, err := resolver(childC[:]) - if bytes.Equal(childC[:], zero[:]) { - continue - } - if err != nil { - return fmt.Errorf("could not find child %x in db: %w", childC, err) - } - // depth is set to 0, the tree isn't rebuilt so it's not a problem - childN, err := verkle.ParseNode(childS, 0) - if err != nil { - return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) - } - if err := checkChildren(childN, resolver); err != nil { - return fmt.Errorf("%x%w", i, err) // write the path to the erroring node - } - } - case *verkle.LeafNode: - // sanity check: ensure at least one value is non-zero - - for i := 0; i < verkle.NodeWidth; i++ { - if len(node.Value(i)) != 0 { - return nil - } - } - return errors.New("both balance and nonce are 0") - case verkle.Empty: - // nothing to do - default: - return fmt.Errorf("unsupported type encountered %v", root) - } - - return nil -} - -func verifyVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true) - defer chaindb.Close() - headBlock := rawdb.ReadHeadBlock(chaindb) - if headBlock == nil { - log.Error("Failed to load head block") - return errors.New("no head block") - } - if ctx.NArg() > 1 { - log.Error("Too many arguments given") - return errors.New("too many arguments") - } - var ( - rootC common.Hash - err error - ) - if ctx.NArg() == 1 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - log.Info("Rebuilding the tree", "root", rootC) - } else { - rootC = headBlock.Root() - log.Info("Rebuilding the tree", "root", rootC, "number", headBlock.NumberU64()) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - if err := checkChildren(root, chaindb.Get); err != nil { - log.Error("Could not rebuild the tree from the database", "err", err) - return err - } - - log.Info("Tree was rebuilt from the database") - return nil -} - -func expandVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true) - defer chaindb.Close() - var ( - rootC common.Hash - keylist [][]byte - err error - ) - if ctx.NArg() >= 2 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - keylist = make([][]byte, 0, ctx.Args().Len()-1) - args := ctx.Args().Slice() - for i := range args[1:] { - key, err := hex.DecodeString(args[i+1]) - log.Info("decoded key", "arg", args[i+1], "key", key) - if err != nil { - return fmt.Errorf("error decoding key #%d: %w", i+1, err) - } - keylist = append(keylist, key) - } - log.Info("Rebuilding the tree", "root", rootC) - } else { - return fmt.Errorf("usage: %s root key1 [key 2...]", ctx.App.Name) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - for i, key := range keylist { - log.Info("Reading key", "index", i, "key", key) - root.Get(key, chaindb.Get) - } - - if err := os.WriteFile("dump.dot", []byte(verkle.ToDot(root)), 0600); err != nil { - log.Error("Failed to dump file", "err", err) - } else { - log.Info("Tree was dumped to file", "file", "dump.dot") - } - return nil -} diff --git a/cmd/geth/version_check.go b/cmd/geth/version_check.go deleted file mode 100644 index 237556788e..0000000000 --- a/cmd/geth/version_check.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "regexp" - "strings" - - "github.com/ethereum/go-ethereum/log" - "github.com/jedisct1/go-minisign" - "github.com/urfave/cli/v2" -) - -var gethPubKeys []string = []string{ - //@holiman, minisign public key FB1D084D39BAEC24 - "RWQk7Lo5TQgd+wxBNZM+Zoy+7UhhMHaWKzqoes9tvSbFLJYZhNTbrIjx", - //minisign public key 138B1CA303E51687 - "RWSHFuUDoxyLEzjszuWZI1xStS66QTyXFFZG18uDfO26CuCsbckX1e9J", - //minisign public key FD9813B2D2098484 - "RWSEhAnSshOY/b+GmaiDkObbCWefsAoavjoLcPjBo1xn71yuOH5I+Lts", -} - -type vulnJson struct { - Name string - Uid string - Summary string - Description string - Links []string - Introduced string - Fixed string - Published string - Severity string - Check string - CVE string -} - -func versionCheck(ctx *cli.Context) error { - url := ctx.String(VersionCheckUrlFlag.Name) - version := ctx.String(VersionCheckVersionFlag.Name) - log.Info("Checking vulnerabilities", "version", version, "url", url) - return checkCurrent(url, version) -} - -func checkCurrent(url, current string) error { - var ( - data []byte - sig []byte - err error - ) - if data, err = fetch(url); err != nil { - return fmt.Errorf("could not retrieve data: %w", err) - } - if sig, err = fetch(fmt.Sprintf("%v.minisig", url)); err != nil { - return fmt.Errorf("could not retrieve signature: %w", err) - } - if err = verifySignature(gethPubKeys, data, sig); err != nil { - return err - } - var vulns []vulnJson - if err = json.Unmarshal(data, &vulns); err != nil { - return err - } - allOk := true - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - if err != nil { - return err - } - if r.MatchString(current) { - allOk = false - fmt.Printf("## Vulnerable to %v (%v)\n\n", vuln.Uid, vuln.Name) - fmt.Printf("Severity: %v\n", vuln.Severity) - fmt.Printf("Summary : %v\n", vuln.Summary) - fmt.Printf("Fixed in: %v\n", vuln.Fixed) - if len(vuln.CVE) > 0 { - fmt.Printf("CVE: %v\n", vuln.CVE) - } - if len(vuln.Links) > 0 { - fmt.Printf("References:\n") - for _, ref := range vuln.Links { - fmt.Printf("\t- %v\n", ref) - } - } - fmt.Println() - } - } - if allOk { - fmt.Println("No vulnerabilities found") - } - return nil -} - -// fetch makes an HTTP request to the given url and returns the response body -func fetch(url string) ([]byte, error) { - if filep := strings.TrimPrefix(url, "file://"); filep != url { - return os.ReadFile(filep) - } - res, err := http.Get(url) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - return body, nil -} - -// verifySignature checks that the sigData is a valid signature of the given -// data, for pubkey GethPubkey -func verifySignature(pubkeys []string, data, sigdata []byte) error { - sig, err := minisign.DecodeSignature(string(sigdata)) - if err != nil { - return err - } - // find the used key - var key *minisign.PublicKey - for _, pubkey := range pubkeys { - pub, err := minisign.NewPublicKey(pubkey) - if err != nil { - // our pubkeys should be parseable - return err - } - if pub.KeyId != sig.KeyId { - continue - } - key = &pub - break - } - if key == nil { - log.Info("Signing key not trusted", "keyid", keyID(sig.KeyId), "error", err) - return errors.New("signature could not be verified") - } - if ok, err := key.Verify(data, sig); !ok || err != nil { - log.Info("Verification failed error", "keyid", keyID(key.KeyId), "error", err) - return errors.New("signature could not be verified") - } - return nil -} - -// keyID turns a binary minisign key ID into a hex string. -// Note: key IDs are printed in reverse byte order. -func keyID(id [8]byte) string { - var rev [8]byte - for i := range id { - rev[len(rev)-1-i] = id[i] - } - return fmt.Sprintf("%X", rev) -} diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go deleted file mode 100644 index fb5d1b2d69..0000000000 --- a/cmd/geth/version_check_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "testing" - - "github.com/jedisct1/go-minisign" -) - -func TestVerification(t *testing.T) { - t.Parallel() - // Signatures generated with `minisign`. Legacy format, not pre-hashed file. - t.Run("minisig-legacy", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // 1. `minisign -S -l -s ./minisign.sec -m data.json -x ./minisig-sigs/vulnerabilities.json.minisig.1 -c "signature from minisign secret key"` - // 2. `minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.2 -c "Here's a comment" -t "Here's a trusted comment"` - // 3. minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.3 -c "One more (untrusted™) comment" -t "Here's a trusted comment" - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs/") - }) - t.Run("minisig-new", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // `minisign -S -s ./minisign.sec -m data.json -x ./minisig-sigs-new/data.json.minisig` - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs-new/") - }) - // Signatures generated with `signify-openbsd` - t.Run("signify-openbsd", func(t *testing.T) { - t.Parallel() - t.Skip("This currently fails, minisign expects 4 lines of data, signify provides only 2") - // For this test, the pubkey is in testdata/vcheck/signifykey.pub - // (the privkey is `signifykey.sec`, if we want to expand this test. Password 'test' ) - // `signify -S -s signifykey.sec -m data.json -x ./signify-sigs/data.json.sig` - pub := "RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/" - testVerification(t, pub, "./testdata/vcheck/signify-sigs/") - }) -} - -func testVerification(t *testing.T, pubkey, sigdir string) { - // Data to verify - data, err := os.ReadFile("./testdata/vcheck/data.json") - if err != nil { - t.Fatal(err) - } - // Signatures, with and without comments, both trusted and untrusted - files, err := os.ReadDir(sigdir) - if err != nil { - t.Fatal(err) - } - if len(files) == 0 { - t.Fatal("Missing tests") - } - for _, f := range files { - sig, err := os.ReadFile(filepath.Join(sigdir, f.Name())) - if err != nil { - t.Fatal(err) - } - err = verifySignature([]string{pubkey}, data, sig) - if err != nil { - t.Fatal(err) - } - } -} - -func versionUint(v string) int { - mustInt := func(s string) int { - a, err := strconv.Atoi(s) - if err != nil { - panic(v) - } - return a - } - components := strings.Split(strings.TrimPrefix(v, "v"), ".") - a := mustInt(components[0]) - b := mustInt(components[1]) - c := mustInt(components[2]) - return a*100*100 + b*100 + c -} - -// TestMatching can be used to check that the regexps are correct -func TestMatching(t *testing.T) { - t.Parallel() - data, _ := os.ReadFile("./testdata/vcheck/vulnerabilities.json") - var vulns []vulnJson - if err := json.Unmarshal(data, &vulns); err != nil { - t.Fatal(err) - } - check := func(version string) { - vFull := fmt.Sprintf("Geth/%v-unstable-15339cf1-20201204/linux-amd64/go1.15.4", version) - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - vulnIntro := versionUint(vuln.Introduced) - vulnFixed := versionUint(vuln.Fixed) - current := versionUint(version) - if err != nil { - t.Fatal(err) - } - if vuln.Name == "Denial of service due to Go CVE-2020-28362" { - // this one is not tied to geth-versions - continue - } - if vulnIntro <= current && vulnFixed > current { - // Should be vulnerable - if !r.MatchString(vFull) { - t.Errorf("Should be vulnerable, version %v, intro: %v, fixed: %v %v %v", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vuln.Check) - } - } else { - if r.MatchString(vFull) { - t.Errorf("Should not be flagged vulnerable, version %v, intro: %v, fixed: %v %v %d %d %d", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) - } - } - } - } - for major := 1; major < 2; major++ { - for minor := 0; minor < 30; minor++ { - for patch := 0; patch < 30; patch++ { - vShort := fmt.Sprintf("v%d.%d.%d", major, minor, patch) - check(vShort) - } - } - } -} - -func TestGethPubKeysParseable(t *testing.T) { - t.Parallel() - for _, pubkey := range gethPubKeys { - _, err := minisign.NewPublicKey(pubkey) - if err != nil { - t.Errorf("Should be parseable") - } - } -} - -func TestKeyID(t *testing.T) { - t.Parallel() - type args struct { - id [8]byte - } - tests := []struct { - name string - args args - want string - }{ - {"@holiman key", args{id: extractKeyId(gethPubKeys[0])}, "FB1D084D39BAEC24"}, - {"second key", args{id: extractKeyId(gethPubKeys[1])}, "138B1CA303E51687"}, - {"third key", args{id: extractKeyId(gethPubKeys[2])}, "FD9813B2D2098484"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := keyID(tt.args.id); got != tt.want { - t.Errorf("keyID() = %v, want %v", got, tt.want) - } - }) - } -} - -func extractKeyId(pubkey string) [8]byte { - p, _ := minisign.NewPublicKey(pubkey) - return p.KeyId -} diff --git a/core/rawdb/database_tablewriter_unix.go b/cmd/keeper/getpayload_wasm.go similarity index 70% rename from core/rawdb/database_tablewriter_unix.go rename to cmd/keeper/getpayload_wasm.go index 8bec5396e8..b912678825 100644 --- a/core/rawdb/database_tablewriter_unix.go +++ b/cmd/keeper/getpayload_wasm.go @@ -1,4 +1,4 @@ -// Copyright 2025 The go-ethereum Authors +// Copyright 2026 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,20 +14,23 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !tinygo -// +build !tinygo +//go:build wasm +// +build wasm -package rawdb +package main import ( - "io" - - "github.com/olekukonko/tablewriter" + "unsafe" ) -// Re-export the real tablewriter types and functions -type Table = tablewriter.Table +//go:wasmimport geth_io len +func hintLen() uint32 + +//go:wasmimport geth_io read +func hintRead(data unsafe.Pointer) -func newTableWriter(w io.Writer) *Table { - return tablewriter.NewWriter(w) +func getInput() []byte { + data := make([]byte, hintLen()) + hintRead(unsafe.Pointer(&data[0])) + return data } diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index ceae65d491..e801fbf6f1 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -21,18 +21,18 @@ require ( github.com/cockroachdb/pebble v1.1.5 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/gnark-crypto v0.18.0 // indirect + github.com/consensys/gnark-crypto v0.18.1 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/emicklei/dot v1.6.2 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab // indirect - github.com/ethereum/go-verkle v0.2.2 // indirect github.com/ferranbt/fastssz v0.1.4 // indirect github.com/gammazero/deque v1.1.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -41,33 +41,34 @@ require ( github.com/gorilla/websocket v1.4.2 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.3.2 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.15.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - golang.org/x/crypto v0.36.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + golang.org/x/crypto v0.44.0 // indirect golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.23.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index 846ad62362..ac17013b7b 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -30,12 +30,10 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/gnark-crypto v0.18.0 h1:vIye/FqI50VeAr0B3dx+YjeIvmc3LWz4yEfbWBpTUf0= -github.com/consensys/gnark-crypto v0.18.0/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= +github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= +github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -51,8 +49,6 @@ github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3 github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -65,6 +61,11 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -88,8 +89,10 @@ github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -99,8 +102,8 @@ github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -112,9 +115,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -123,8 +123,6 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= @@ -149,15 +147,13 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4= github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -168,11 +164,21 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -184,15 +190,15 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -207,13 +213,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -229,8 +235,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/cmd/keeper/main.go b/cmd/keeper/main.go index 9b459f6f36..df6881acbf 100644 --- a/cmd/keeper/main.go +++ b/cmd/keeper/main.go @@ -17,6 +17,7 @@ package main import ( + "context" "fmt" "os" "runtime/debug" @@ -52,7 +53,7 @@ func main() { } vmConfig := vm.Config{} - crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(chainConfig, vmConfig, payload.Block, payload.Witness) + crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(context.Background(), chainConfig, vmConfig, payload.Block, payload.Witness) if err != nil { fmt.Fprintf(os.Stderr, "stateless self-validation failed: %v\n", err) os.Exit(10) diff --git a/cmd/keeper/stubs.go b/cmd/keeper/stubs.go index 04a3bc735b..407a21a145 100644 --- a/cmd/keeper/stubs.go +++ b/cmd/keeper/stubs.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !example && !ziren +//go:build !example && !ziren && !wasm +// +build !example,!ziren,!wasm package main diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3e337a3d00..995724e6fc 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -57,6 +57,8 @@ const ( importBatchSize = 2500 ) +type EraFileFormat int + // ErrImportInterrupted is returned when the user interrupts the import process. var ErrImportInterrupted = errors.New("interrupted") @@ -250,7 +252,7 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string) error { +func ImportHistory(chain *core.BlockChain, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { return errors.New("history import only supported when starting from genesis") } @@ -263,42 +265,49 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { return fmt.Errorf("unable to read checksums.txt: %w", err) } if len(checksums) != len(entries) { - return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", + len(checksums), len(entries)) } + var ( start = time.Now() reported = time.Now() imported = 0 h = sha256.New() - buf = bytes.NewBuffer(nil) + scratch = bytes.NewBuffer(nil) ) - for i, filename := range entries { + + for i, file := range entries { err := func() error { - f, err := os.Open(filepath.Join(dir, filename)) + path := filepath.Join(dir, file) + + // validate against checksum file in directory + f, err := os.Open(path) if err != nil { - return fmt.Errorf("unable to open era: %w", err) + return fmt.Errorf("open %s: %w", path, err) } defer f.Close() - - // Validate checksum. if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to recalculate checksum: %w", err) - } - if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { - return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + return fmt.Errorf("checksum %s: %w", path, err) } + got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex() + want := checksums[i] h.Reset() - buf.Reset() + scratch.Reset() + if got != want { + return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) + } // Import all block data from Era1. - e, err := era.From(f) + e, err := from(f) if err != nil { return fmt.Errorf("error opening era: %w", err) } - it, err := era.NewIterator(e) + it, err := e.Iterator() if err != nil { - return fmt.Errorf("error making era reader: %w", err) + return fmt.Errorf("error creating iterator: %w", err) } + for it.Next() { block, err := it.Block() if err != nil { @@ -311,26 +320,28 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) } - encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) - if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil { + enc := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) + if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } - imported += 1 + imported++ - // Give the user some feedback that something is happening. if time.Since(reported) >= 8*time.Second { - log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Importing Era files", "head", it.Number(), "imported", imported, + "elapsed", common.PrettyDuration(time.Since(start))) imported = 0 reported = time.Now() } } + if err := it.Error(); err != nil { + return err + } return nil }() if err != nil { return err } } - return nil } @@ -389,7 +400,6 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return err } defer fh.Close() - var writer io.Writer = fh if strings.HasSuffix(fn, ".gz") { writer = gzip.NewWriter(writer) @@ -405,7 +415,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las // ExportHistory exports blockchain history into the specified directory, // following the Era format. -func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { +func ExportHistory(bc *core.BlockChain, dir string, first, last uint64, newBuilder func(io.Writer) era.Builder, filename func(network string, epoch int, lastBlockHash common.Hash) string) error { log.Info("Exporting blockchain history", "dir", dir) if head := bc.CurrentBlock().Number.Uint64(); head < last { log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) @@ -418,76 +428,100 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("error creating output directory: %w", err) } + var ( start = time.Now() reported = time.Now() h = sha256.New() buf = bytes.NewBuffer(nil) + td = new(big.Int) checksums []string ) - td := new(big.Int) - for i := uint64(0); i < first; i++ { - td.Add(td, bc.GetHeaderByNumber(i).Difficulty) + + // Compute initial TD by accumulating difficulty from genesis to first-1. + // This is necessary because TD is no longer stored in the database. Only + // compute if a segment of the export is pre-merge. + b := bc.GetBlockByNumber(first) + if b == nil { + return fmt.Errorf("block #%d not found", first) + } + if first > 0 && b.Difficulty().Sign() != 0 { + log.Info("Computing initial total difficulty", "from", 0, "to", first-1) + for i := uint64(0); i < first; i++ { + b := bc.GetBlockByNumber(i) + if b == nil { + return fmt.Errorf("block #%d not found while computing initial TD", i) + } + td.Add(td, b.Difficulty()) + } + log.Info("Initial total difficulty computed", "td", td) } - for i := first; i <= last; i += step { - err := func() error { - filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) - f, err := os.Create(filename) + + for batch := first; batch <= last; batch += uint64(era.MaxSize) { + idx := int(batch / uint64(era.MaxSize)) + tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{})) + + if err := func() error { + f, err := os.Create(tmpPath) if err != nil { - return fmt.Errorf("could not create era file: %w", err) + return err } defer f.Close() - w := era.NewBuilder(f) - for j := uint64(0); j < step && j <= last-i; j++ { - var ( - n = i + j - block = bc.GetBlockByNumber(n) - ) + builder := newBuilder(f) + + for j := uint64(0); j < uint64(era.MaxSize) && batch+j <= last; j++ { + n := batch + j + block := bc.GetBlockByNumber(n) if block == nil { - return fmt.Errorf("export failed on #%d: not found", n) + return fmt.Errorf("block #%d not found", n) } - receipts := bc.GetReceiptsByHash(block.Hash()) - if receipts == nil { - return fmt.Errorf("export failed on #%d: receipts not found", n) + receipt := bc.GetReceiptsByHash(block.Hash()) + if receipt == nil { + return fmt.Errorf("receipts for #%d missing", n) } - td.Add(td, block.Difficulty()) - if err := w.Add(block, receipts, new(big.Int).Set(td)); err != nil { + + // For pre-merge blocks, pass accumulated TD. + // For post-merge blocks (difficulty == 0), pass nil TD. + var blockTD *big.Int + if block.Difficulty().Sign() != 0 { + td.Add(td, block.Difficulty()) + blockTD = new(big.Int).Set(td) + } + + if err := builder.Add(block, receipt, blockTD); err != nil { return err } } - root, err := w.Finalize() + id, err := builder.Finalize() if err != nil { - return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + return err } - // Set correct filename with root. - os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) - - // Compute checksum of entire Era1. if _, err := f.Seek(0, io.SeekStart); err != nil { return err } + h.Reset() + buf.Reset() if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to calculate checksum: %w", err) + return err } checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) - h.Reset() - buf.Reset() - return nil - }() - if err != nil { + + // Close before rename. It's required on Windows. + f.Close() + final := filepath.Join(dir, filename(network, idx, id)) + return os.Rename(tmpPath, final) + }(); err != nil { return err } + if time.Since(reported) >= 8*time.Second { - log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("export progress", "exported", batch, "elapsed", common.PrettyDuration(time.Since(start))) reported = time.Now() } } - os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) - - log.Info("Exported blockchain to", "dir", dir) - + _ = os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f22db37971..4f3ad054b6 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -114,7 +114,7 @@ var ( Usage: "Root directory for era1 history (default = inside ancient/chain)", Category: flags.EthCategory, } - MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ + MinFreeDiskSpaceFlag = &cli.IntFlag{ Name: "datadir.minfreedisk", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", Category: flags.EthCategory, @@ -137,7 +137,7 @@ var ( } NetworkIdFlag = &cli.Uint64Flag{ Name: "networkid", - Usage: "Explicitly set network id (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", + Usage: "Explicitly set network ID (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", Value: ethconfig.Defaults.NetworkId, Category: flags.EthCategory, } @@ -295,6 +295,18 @@ var ( Value: ethconfig.Defaults.StateHistory, Category: flags.StateCategory, } + TrienodeHistoryFlag = &cli.Int64Flag{ + Name: "history.trienode", + Usage: "Number of recent blocks to retain trienode history for, only relevant in state.scheme=path (default/negative = disabled, 0 = entire chain)", + Value: ethconfig.Defaults.TrienodeHistory, + Category: flags.StateCategory, + } + TrienodeHistoryFullValueCheckpointFlag = &cli.UintFlag{ + Name: "history.trienode.full-value-checkpoint", + Usage: "The frequency of full-value encoding. Every n-th node is stored in full-value format; all other nodes are stored as diffs relative to their predecessor", + Value: uint(ethconfig.Defaults.NodeFullValueCheckpoint), + Category: flags.StateCategory, + } TransactionHistoryFlag = &cli.Uint64Flag{ Name: "history.transactions", Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", @@ -553,6 +565,11 @@ var ( Usage: "0x prefixed public address for the pending block producer (not used for actual block production)", Category: flags.MinerCategory, } + MinerMaxBlobsFlag = &cli.IntFlag{ + Name: "miner.maxblobs", + Usage: "Maximum number of blobs per block (falls back to protocol maximum if unspecified)", + Category: flags.MinerCategory, + } // Account settings PasswordFileFlag = &cli.PathFlag{ @@ -631,6 +648,12 @@ var ( Value: ethconfig.Defaults.TxSyncMaxTimeout, Category: flags.APICategory, } + RPCGlobalRangeLimitFlag = &cli.Uint64Flag{ + Name: "rpc.rangelimit", + Usage: "Maximum block range (end - begin) allowed for range queries (0 = unlimited)", + Value: ethconfig.Defaults.RangeLimit, + Category: flags.APICategory, + } // Authenticated RPC HTTP settings AuthListenFlag = &cli.StringFlag{ Name: "authrpc.addr", @@ -667,6 +690,12 @@ var ( Usage: "Disables db compaction after import", Category: flags.LoggingCategory, } + LogSlowBlockFlag = &cli.DurationFlag{ + Name: "debug.logslowblock", + Usage: "Block execution time threshold beyond which detailed statistics will be logged (0 logs all blocks, negative means disable)", + Value: ethconfig.Defaults.SlowBlockThreshold, + Category: flags.LoggingCategory, + } // MISC settings SyncTargetFlag = &cli.StringFlag{ @@ -859,14 +888,14 @@ var ( Aliases: []string{"discv4"}, Usage: "Enables the V4 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV4, } DiscoveryV5Flag = &cli.BoolFlag{ Name: "discovery.v5", Aliases: []string{"discv5"}, Usage: "Enables the V5 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV5, } NetrestrictFlag = &cli.StringFlag{ Name: "netrestrict", @@ -1013,6 +1042,55 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Value: metrics.DefaultConfig.InfluxDBOrganization, Category: flags.MetricsCategory, } + + // RPC Telemetry + RPCTelemetryFlag = &cli.BoolFlag{ + Name: "rpc.telemetry", + Usage: "Enable RPC telemetry", + Category: flags.APICategory, + } + + RPCTelemetryEndpointFlag = &cli.StringFlag{ + Name: "rpc.telemetry.endpoint", + Usage: "Defines where RPC telemetry is sent (e.g., http://localhost:4318)", + Category: flags.APICategory, + } + + RPCTelemetryUserFlag = &cli.StringFlag{ + Name: "rpc.telemetry.username", + Usage: "HTTP Basic Auth username for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryPasswordFlag = &cli.StringFlag{ + Name: "rpc.telemetry.password", + Usage: "HTTP Basic Auth password for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryInstanceIDFlag = &cli.StringFlag{ + Name: "rpc.telemetry.instance-id", + Usage: "OpenTelemetry instance ID", + Category: flags.APICategory, + } + + RPCTelemetryTagsFlag = &cli.StringFlag{ + Name: "rpc.telemetry.tags", + Usage: "Comma-separated tags (key/values) added as attributes to the OpenTelemetry resource struct", + Category: flags.APICategory, + } + + RPCTelemetrySampleRatioFlag = &cli.Float64Flag{ + Name: "rpc.telemetry.sample-ratio", + Usage: "Defines the sampling ratio for RPC telemetry (0.0 to 1.0)", + Value: 1.0, + Category: flags.APICategory, + } + // Era flags are a group of flags related to the era archive format. + EraFormatFlag = &cli.StringFlag{ + Name: "era.format", + Usage: "Archive format: 'era1' or 'erae'", + } ) var ( @@ -1363,13 +1441,17 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.MaxPendingPeers = ctx.Int(MaxPendingPeersFlag.Name) } if ctx.IsSet(NoDiscoverFlag.Name) { - cfg.NoDiscovery = true + cfg.NoDiscovery = ctx.Bool(NoDiscoverFlag.Name) } flags.CheckExclusive(ctx, DiscoveryV4Flag, NoDiscoverFlag) flags.CheckExclusive(ctx, DiscoveryV5Flag, NoDiscoverFlag) - cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) - cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + if ctx.IsSet(DiscoveryV4Flag.Name) { + cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) + } + if ctx.IsSet(DiscoveryV5Flag.Name) { + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + } if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) @@ -1399,6 +1481,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setNodeUserIdent(ctx, cfg) SetDataDir(ctx, cfg) setSmartCard(ctx, cfg) + setOpenTelemetry(ctx, cfg) if ctx.IsSet(JWTSecretFlag.Name) { cfg.JWTSecret = ctx.String(JWTSecretFlag.Name) @@ -1415,7 +1498,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.KeyStoreDir = ctx.String(KeyStoreDirFlag.Name) } if ctx.IsSet(DeveloperFlag.Name) { - cfg.UseLightweightKDF = true + cfg.UseLightweightKDF = ctx.Bool(DeveloperFlag.Name) } if ctx.IsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.Bool(LightKDFFlag.Name) @@ -1466,6 +1549,33 @@ func setSmartCard(ctx *cli.Context, cfg *node.Config) { cfg.SmartCardDaemonPath = path } +func setOpenTelemetry(ctx *cli.Context, cfg *node.Config) { + tcfg := &cfg.OpenTelemetry + if ctx.IsSet(RPCTelemetryFlag.Name) { + tcfg.Enabled = ctx.Bool(RPCTelemetryFlag.Name) + } + if ctx.IsSet(RPCTelemetryEndpointFlag.Name) { + tcfg.Endpoint = ctx.String(RPCTelemetryEndpointFlag.Name) + } + if ctx.IsSet(RPCTelemetryUserFlag.Name) { + tcfg.AuthUser = ctx.String(RPCTelemetryUserFlag.Name) + } + if ctx.IsSet(RPCTelemetryPasswordFlag.Name) { + tcfg.AuthPassword = ctx.String(RPCTelemetryPasswordFlag.Name) + } + if ctx.IsSet(RPCTelemetryInstanceIDFlag.Name) { + tcfg.InstanceID = ctx.String(RPCTelemetryInstanceIDFlag.Name) + } + if ctx.IsSet(RPCTelemetryTagsFlag.Name) { + tcfg.Tags = ctx.String(RPCTelemetryTagsFlag.Name) + } + tcfg.SampleRatio = ctx.Float64(RPCTelemetrySampleRatioFlag.Name) + + if tcfg.Endpoint != "" && !tcfg.Enabled { + log.Warn(fmt.Sprintf("OpenTelemetry endpoint configured but telemetry is not enabled, use --%s to enable.", RPCTelemetryFlag.Name)) + } +} + func SetDataDir(ctx *cli.Context, cfg *node.Config) { switch { case ctx.IsSet(DataDirFlag.Name): @@ -1571,6 +1681,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit") cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name) } + if ctx.IsSet(MinerMaxBlobsFlag.Name) { + cfg.MaxBlobsPerBlock = ctx.Int(MinerMaxBlobsFlag.Name) + } } func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { @@ -1603,8 +1716,8 @@ func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { - // Avoid conflicting network flags, don't allow network id override on preset networks - flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, NetworkIdFlag, OverrideGenesisFlag) + // Avoid conflicting network flags + flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, OverrideGenesisFlag) flags.CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer // Set configurations from CLI flags @@ -1651,9 +1764,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } } - if ctx.IsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.Uint64(NetworkIdFlag.Name) - } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 } @@ -1674,8 +1784,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } - // Read the value from the flag no matter if it's set or not. - cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) + if ctx.IsSet(CachePreimagesFlag.Name) { + cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) + } if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") @@ -1683,6 +1794,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } + if ctx.IsSet(TrienodeHistoryFlag.Name) { + cfg.TrienodeHistory = ctx.Int64(TrienodeHistoryFlag.Name) + } + if ctx.IsSet(TrienodeHistoryFullValueCheckpointFlag.Name) { + cfg.NodeFullValueCheckpoint = uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)) + } if ctx.IsSet(StateSchemeFlag.Name) { cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } @@ -1708,7 +1825,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name) } if ctx.IsSet(LogNoHistoryFlag.Name) { - cfg.LogNoHistory = true + cfg.LogNoHistory = ctx.Bool(LogNoHistoryFlag.Name) + } + if ctx.IsSet(LogSlowBlockFlag.Name) { + cfg.SlowBlockThreshold = ctx.Duration(LogSlowBlockFlag.Name) } if ctx.IsSet(LogExportCheckpointsFlag.Name) { cfg.LogExportCheckpoints = ctx.String(LogExportCheckpointsFlag.Name) @@ -1734,6 +1854,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(RPCTxSyncMaxTimeoutFlag.Name) { cfg.TxSyncMaxTimeout = ctx.Duration(RPCTxSyncMaxTimeoutFlag.Name) } + if ctx.IsSet(RPCGlobalRangeLimitFlag.Name) { + cfg.RangeLimit = ctx.Uint64(RPCGlobalRangeLimitFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 { // If snap-sync is requested, this flag is also required if cfg.SyncMode == ethconfig.SnapSync { @@ -1903,10 +2026,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = genesis default: - if cfg.NetworkId == 1 { + if ctx.Uint64(NetworkIdFlag.Name) == 1 { SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) } } + if ctx.IsSet(NetworkIdFlag.Name) { + // Typically it's best to automatically set the network ID to the chainID, + // by not passing the --networkid flag at all. Emit a warning when set + // explicitly in case overriding the network ID is not the user's intention. + id := ctx.Uint64(NetworkIdFlag.Name) + log.Warn("Setting network ID with command-line flag", "id", id) + cfg.NetworkId = id + } // Set any dangling config values if ctx.String(CryptoKZGFlag.Name) != "gokzg" && ctx.String(CryptoKZGFlag.Name) != "ckzg" { Fatalf("--%s flag must be 'gokzg' or 'ckzg'", CryptoKZGFlag.Name) @@ -2070,6 +2201,7 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf filterSystem := filters.NewFilterSystem(backend, filters.Config{ LogCacheSize: ethcfg.FilterLogCacheSize, LogQueryLimit: ethcfg.LogQueryLimit, + RangeLimit: ethcfg.RangeLimit, }) stack.RegisterAPIs([]rpc.API{{ Namespace: "eth", @@ -2272,15 +2404,18 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh Fatalf("%v", err) } options := &core.BlockChainConfig{ - TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), - TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", - TrieTimeLimit: ethconfig.Defaults.TrieTimeout, - SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.Bool(CachePreimagesFlag.Name), - StateScheme: scheme, - StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, + NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), + TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, + ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", + TrieTimeLimit: ethconfig.Defaults.TrieTimeout, + SnapshotLimit: ethconfig.Defaults.SnapshotCache, + Preimages: ctx.Bool(CachePreimagesFlag.Name), + StateScheme: scheme, + StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrienodeHistory: ctx.Int64(TrienodeHistoryFlag.Name), + NodeFullValueCheckpoint: uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)), + // Disable transaction indexing/unindexing. TxIndexer: nil, @@ -2292,6 +2427,13 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh // Enable state size tracking if enabled StateSizeTracking: ctx.Bool(StateSizeTrackingFlag.Name), + + // Configure the slow block statistic logger (disabled by default) + SlowBlockThreshold: ethconfig.Defaults.SlowBlockThreshold, + } + // Only enable slow block logging if the flag was explicitly set + if ctx.IsSet(LogSlowBlockFlag.Name) { + options.SlowBlockThreshold = ctx.Duration(LogSlowBlockFlag.Name) } if options.ArchiveMode && !options.Preimages { options.Preimages = true diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index d2024f8611..9dbbc1c551 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -33,6 +33,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -44,136 +46,148 @@ var ( ) func TestHistoryImportAndExport(t *testing.T) { - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, - } - signer = types.LatestSigner(genesis.Config) - ) - - // Generate chain. - db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { - if i == 0 { - return - } - tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ - ChainID: genesis.Config.ChainID, - Nonce: uint64(i - 1), - GasTipCap: common.Big0, - GasFeeCap: g.PrevBlock(0).BaseFee(), - Gas: 50000, - To: &common.Address{0xaa}, - Value: big.NewInt(int64(i)), - Data: nil, - AccessList: nil, - }) - if err != nil { - t.Fatalf("error creating tx: %v", err) - } - g.AddTx(tx) - }) + for _, tt := range []struct { + name string + builder func(io.Writer) era.Builder + filename func(network string, epoch int, root common.Hash) string + from func(f era.ReadAtSeekCloser) (era.Era, error) + }{ + {"era1", onedb.NewBuilder, onedb.Filename, onedb.From}, + {"erae", execdb.NewBuilder, execdb.Filename, execdb.From}, + } { + t.Run(tt.name, func(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) - // Initialize BlockChain. - chain, err := core.NewBlockChain(db, nil, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if _, err := chain.InsertChain(blocks); err != nil { - t.Fatalf("error inserting chain: %v", err) - } + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) - // Make temp directory for era files. - dir := t.TempDir() + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, ethash.NewFaker(), nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error inserting chain: %v", err) + } - // Export history to temp directory. - if err := ExportHistory(chain, dir, 0, count, step); err != nil { - t.Fatalf("error exporting history: %v", err) - } + // Make temp directory for era files. + dir := t.TempDir() - // Read checksums. - b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) - if err != nil { - t.Fatalf("failed to read checksums: %v", err) - } - checksums := strings.Split(string(b), "\n") + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, tt.builder, tt.filename); err != nil { + t.Fatalf("error exporting history: %v", err) + } - // Verify each Era. - entries, _ := era.ReadDir(dir, "mainnet") - for i, filename := range entries { - func() { - f, err := os.Open(filepath.Join(dir, filename)) + // Read checksums. + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) if err != nil { - t.Fatalf("error opening era file: %v", err) - } - var ( - h = sha256.New() - buf = bytes.NewBuffer(nil) - ) - if _, err := io.Copy(h, f); err != nil { - t.Fatalf("unable to recalculate checksum: %v", err) + t.Fatalf("failed to read checksums: %v", err) } - if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { - t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(filepath.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := tt.from(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := e.Iterator() + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() } - e, err := era.From(f) + + // Now import Era. + db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) if err != nil { - t.Fatalf("error opening era: %v", err) + panic(err) } - defer e.Close() - it, err := era.NewIterator(e) + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) + imported, err := core.NewBlockChain(db2, nil, genesis, ethash.NewFaker(), nil) if err != nil { - t.Fatalf("error making era reader: %v", err) + t.Fatalf("unable to initialize chain: %v", err) } - for j := 0; it.Next(); j++ { - n := i*int(step) + j - if it.Error() != nil { - t.Fatalf("error reading block entry %d: %v", n, it.Error()) - } - block, receipts, err := it.BlockAndReceipts() - if err != nil { - t.Fatalf("error reading block entry %d: %v", n, err) - } - want := chain.GetBlockByNumber(uint64(n)) - if want, got := uint64(n), block.NumberU64(); want != got { - t.Fatalf("blocks out of order: want %d, got %d", want, got) - } - if want.Hash() != block.Hash() { - t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) - } - if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { - t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) - } - if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { - t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) - } - if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { - t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) - } + if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil { + t.Fatalf("failed to import chain: %v", err) } - }() - } - - // Now import Era. - db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) - if err != nil { - panic(err) - } - t.Cleanup(func() { - db2.Close() - }) - - genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) - imported, err := core.NewBlockChain(db2, nil, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if err := ImportHistory(imported, dir, "mainnet"); err != nil { - t.Fatalf("failed to import chain: %v", err) - } - if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { - t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } + }) } } diff --git a/cmd/workload/README.md b/cmd/workload/README.md index 1b84dd05db..ee1d6acbc9 100644 --- a/cmd/workload/README.md +++ b/cmd/workload/README.md @@ -34,4 +34,5 @@ the following commands (in this directory) against a synced mainnet node: > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545 > go run . tracegen --trace-tests queries/trace_mainnet.json --trace-start 4000000 --trace-end 4000100 http://host:8545 +> go run . proofgen --proof-tests queries/proof_mainnet.json --proof-states 3000 http://host:8545 ``` diff --git a/cmd/workload/main.go b/cmd/workload/main.go index 8ac0e5b6cb..4ee894e962 100644 --- a/cmd/workload/main.go +++ b/cmd/workload/main.go @@ -48,6 +48,7 @@ func init() { historyGenerateCommand, filterGenerateCommand, traceGenerateCommand, + proofGenerateCommand, filterPerfCommand, filterFuzzCommand, } diff --git a/cmd/workload/prooftest.go b/cmd/workload/prooftest.go new file mode 100644 index 0000000000..dcc063d30e --- /dev/null +++ b/cmd/workload/prooftest.go @@ -0,0 +1,105 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/urfave/cli/v2" +) + +// proofTest is the content of a state-proof test. +type proofTest struct { + BlockNumbers []uint64 `json:"blockNumbers"` + Addresses [][]common.Address `json:"addresses"` + StorageKeys [][][]string `json:"storageKeys"` + Results [][]common.Hash `json:"results"` +} + +type proofTestSuite struct { + cfg testConfig + tests proofTest + invalidDir string +} + +func newProofTestSuite(cfg testConfig, ctx *cli.Context) *proofTestSuite { + s := &proofTestSuite{ + cfg: cfg, + invalidDir: ctx.String(proofTestInvalidOutputFlag.Name), + } + if err := s.loadTests(); err != nil { + exit(err) + } + return s +} + +func (s *proofTestSuite) loadTests() error { + file, err := s.cfg.fsys.Open(s.cfg.proofTestFile) + if err != nil { + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.proofTestFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open proofTestFile: %v", err) + } + } + defer file.Close() + if err := json.NewDecoder(file).Decode(&s.tests); err != nil { + return fmt.Errorf("invalid JSON in %s: %v", s.cfg.proofTestFile, err) + } + if len(s.tests.BlockNumbers) == 0 { + return fmt.Errorf("proofTestFile %s has no test data", s.cfg.proofTestFile) + } + return nil +} + +func (s *proofTestSuite) allTests() []workloadTest { + return []workloadTest{ + newArchiveWorkloadTest("Proof/GetProof", s.getProof), + } +} + +func (s *proofTestSuite) getProof(t *utesting.T) { + ctx := context.Background() + for i, blockNumber := range s.tests.BlockNumbers { + for j := 0; j < len(s.tests.Addresses[i]); j++ { + res, err := s.cfg.client.Geth.GetProof(ctx, s.tests.Addresses[i][j], s.tests.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + t.Errorf("State proving fails, blockNumber: %d, address: %x, keys: %v, err: %v\n", blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " "), err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + t.Fatalf("State proving fails: error %v", err) + continue + } + if crypto.Keccak256Hash(blob) != s.tests.Results[i][j] { + t.Errorf("State proof mismatch, %d, number: %d, address: %x, keys: %v: invalid result", i, blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " ")) + } + } + } +} diff --git a/cmd/workload/prooftestgen.go b/cmd/workload/prooftestgen.go new file mode 100644 index 0000000000..5d92eea114 --- /dev/null +++ b/cmd/workload/prooftestgen.go @@ -0,0 +1,355 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/eth/tracers/native" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/testrand" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + proofGenerateCommand = &cli.Command{ + Name: "proofgen", + Usage: "Generates tests for state proof verification", + ArgsUsage: "", + Action: generateProofTests, + Flags: []cli.Flag{ + proofTestFileFlag, + proofTestResultOutputFlag, + proofTestStatesFlag, + proofTestStartBlockFlag, + proofTestEndBlockFlag, + }, + } + + proofTestFileFlag = &cli.StringFlag{ + Name: "proof-tests", + Usage: "JSON file containing proof test queries", + Value: "proof_tests.json", + Category: flags.TestingCategory, + } + proofTestResultOutputFlag = &cli.StringFlag{ + Name: "proof-output", + Usage: "Folder containing detailed trace output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStatesFlag = &cli.Int64Flag{ + Name: "proof-states", + Usage: "Number of states to generate proof against", + Value: 10000, + Category: flags.TestingCategory, + } + proofTestInvalidOutputFlag = &cli.StringFlag{ + Name: "proof-invalid", + Usage: "Folder containing the mismatched state proof output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStartBlockFlag = &cli.Uint64Flag{ + Name: "proof-start", + Usage: "The number of starting block for proof verification (included)", + Category: flags.TestingCategory, + } + proofTestEndBlockFlag = &cli.Uint64Flag{ + Name: "proof-end", + Usage: "The number of ending block for proof verification (excluded)", + Category: flags.TestingCategory, + } +) + +type proofGenerator func(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) + +func genAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + nAccounts int + ctx = context.Background() + start = time.Now() + ) + chainID, err := cli.Eth.ChainID(ctx) + if err != nil { + return nil, nil, nil, err + } + signer := types.LatestSignerForChainID(chainID) + + for { + if nAccounts >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + keys [][]string + gather = func(address common.Address) { + addresses = append(addresses, address) + keys = append(keys, nil) + nAccounts++ + } + ) + for _, tx := range block.Transactions() { + if nAccounts >= number { + break + } + sender, err := signer.Sender(tx) + if err != nil { + log.Error("Failed to resolve the sender address", "hash", tx.Hash(), "err", err) + continue + } + gather(sender) + + if tx.To() != nil { + gather(*tx.To()) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for account proof", "blocks", len(blockNumbers), "accounts", nAccounts, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genNonExistentAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + total int + ) + for i := 0; i < number/5; i++ { + var ( + addresses []common.Address + keys [][]string + blockNumber = uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + ) + for j := 0; j < 5; j++ { + addresses = append(addresses, testrand.Address()) + keys = append(keys, nil) + } + total += len(addresses) + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for non-existing account proof", "blocks", len(blockNumbers), "accounts", total) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genStorageProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + + nAccounts int + nStorages int + start = time.Now() + ) + for { + if nAccounts+nStorages >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + slots [][]string + tracer = "prestateTracer" + configBlob, _ = json.Marshal(native.PrestateTracerConfig{ + DiffMode: false, + DisableCode: true, + DisableStorage: false, + }) + ) + for _, tx := range block.Transactions() { + if nAccounts+nStorages >= number { + break + } + if tx.To() == nil { + continue + } + ret, err := cli.Geth.TraceTransaction(context.Background(), tx.Hash(), &tracers.TraceConfig{ + Tracer: &tracer, + TracerConfig: configBlob, + }) + if err != nil { + log.Error("Failed to trace the transaction", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + blob, err := json.Marshal(ret) + if err != nil { + log.Error("Failed to marshal data", "err", err) + continue + } + var accounts map[common.Address]*types.Account + if err := json.Unmarshal(blob, &accounts); err != nil { + log.Error("Failed to decode trace result", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + for addr, account := range accounts { + if len(account.Storage) == 0 { + continue + } + addresses = append(addresses, addr) + nAccounts += 1 + + var keys []string + for k := range account.Storage { + keys = append(keys, k.Hex()) + } + nStorages += len(keys) + + var emptyKeys []string + for i := 0; i < 3; i++ { + emptyKeys = append(emptyKeys, testrand.Hash().Hex()) + } + nStorages += len(emptyKeys) + + slots = append(slots, append(keys, emptyKeys...)) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, slots) + } + log.Info("Generated tests for storage proof", "blocks", len(blockNumbers), "accounts", nAccounts, "storages", nStorages, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genProofRequests(cli *client, startBlock, endBlock uint64, states int) (*proofTest, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + ) + ratio := []float64{0.2, 0.1, 0.7} + for i, fn := range []proofGenerator{genAccountProof, genNonExistentAccountProof, genStorageProof} { + numbers, addresses, keys, err := fn(cli, startBlock, endBlock, int(float64(states)*ratio[i])) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, numbers...) + accountAddresses = append(accountAddresses, addresses...) + storageKeys = append(storageKeys, keys...) + } + return &proofTest{ + BlockNumbers: blockNumbers, + Addresses: accountAddresses, + StorageKeys: storageKeys, + }, nil +} + +func generateProofTests(clictx *cli.Context) error { + var ( + client = makeClient(clictx) + ctx = context.Background() + states = clictx.Int(proofTestStatesFlag.Name) + outputFile = clictx.String(proofTestFileFlag.Name) + outputDir = clictx.String(proofTestResultOutputFlag.Name) + startBlock = clictx.Uint64(proofTestStartBlockFlag.Name) + endBlock = clictx.Uint64(proofTestEndBlockFlag.Name) + ) + head, err := client.Eth.BlockNumber(ctx) + if err != nil { + exit(err) + } + if startBlock > head || endBlock > head { + return fmt.Errorf("chain is out of proof range, head %d, start: %d, limit: %d", head, startBlock, endBlock) + } + if endBlock == 0 { + endBlock = head + } + log.Info("Generating proof states", "startBlock", startBlock, "endBlock", endBlock, "states", states) + + test, err := genProofRequests(client, startBlock, endBlock, states) + if err != nil { + exit(err) + } + for i, blockNumber := range test.BlockNumbers { + var hashes []common.Hash + for j := 0; j < len(test.Addresses[i]); j++ { + res, err := client.Geth.GetProof(ctx, test.Addresses[i][j], test.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + log.Error("Failed to prove the state", "number", blockNumber, "address", test.Addresses[i][j], "slots", len(test.StorageKeys[i][j]), "err", err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + return err + } + hashes = append(hashes, crypto.Keccak256Hash(blob)) + + writeStateProof(outputDir, blockNumber, test.Addresses[i][j], res) + } + test.Results = append(test.Results, hashes) + } + writeJSON(outputFile, test) + return nil +} + +func writeStateProof(dir string, blockNumber uint64, address common.Address, result any) { + if dir == "" { + return + } + // Ensure the directory exists + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + exit(fmt.Errorf("failed to create directories: %w", err)) + } + fname := fmt.Sprintf("%d-%x", blockNumber, address) + name := filepath.Join(dir, fname) + file, err := os.Create(name) + if err != nil { + exit(fmt.Errorf("error creating %s: %v", name, err)) + return + } + defer file.Close() + + data, _ := json.MarshalIndent(result, "", " ") + _, err = file.Write(data) + if err != nil { + exit(fmt.Errorf("error writing %s: %v", name, err)) + return + } +} diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index 25dc17a49e..80cbd15352 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -50,7 +50,9 @@ var ( filterQueryFileFlag, historyTestFileFlag, traceTestFileFlag, + proofTestFileFlag, traceTestInvalidOutputFlag, + proofTestInvalidOutputFlag, }, } testPatternFlag = &cli.StringFlag{ @@ -95,6 +97,7 @@ type testConfig struct { historyTestFile string historyPruneBlock *uint64 traceTestFile string + proofTestFile string } var errPrunedHistory = errors.New("attempt to access pruned history") @@ -145,6 +148,12 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_mainnet.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_mainnet.json" + } + cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber case ctx.Bool(testSepoliaFlag.Name): @@ -164,6 +173,12 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_sepolia.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_sepolia.json" + } + cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber default: @@ -171,6 +186,7 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) cfg.historyTestFile = ctx.String(historyTestFileFlag.Name) cfg.traceTestFile = ctx.String(traceTestFileFlag.Name) + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) } return cfg } @@ -222,11 +238,13 @@ func runTestCmd(ctx *cli.Context) error { filterSuite := newFilterTestSuite(cfg) historySuite := newHistoryTestSuite(cfg) traceSuite := newTraceTestSuite(cfg, ctx) + proofSuite := newProofTestSuite(cfg, ctx) // Filter test cases. tests := filterSuite.allTests() tests = append(tests, historySuite.allTests()...) tests = append(tests, traceSuite.allTests()...) + tests = append(tests, proofSuite.allTests()...) utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool { if t.Slow && !ctx.Bool(testSlowFlag.Name) { diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index a18a6d18ee..578da1cf49 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -8,6 +8,7 @@ package bitutil import ( + "crypto/subtle" "runtime" "unsafe" ) @@ -17,46 +18,16 @@ const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" | // XORBytes xors the bytes in a and b. The destination is assumed to have enough // space. Returns the number of bytes xor'd. +// +// If dst does not have length at least n, +// XORBytes panics without writing anything to dst. +// +// dst and x or y may overlap exactly or not at all, +// otherwise XORBytes may panic. +// +// Deprecated: use crypto/subtle.XORBytes func XORBytes(dst, a, b []byte) int { - if supportsUnaligned { - return fastXORBytes(dst, a, b) - } - return safeXORBytes(dst, a, b) -} - -// fastXORBytes xors in bulk. It only works on architectures that support -// unaligned read/writes. -func fastXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - w := n / wordSize - if w > 0 { - dw := *(*[]uintptr)(unsafe.Pointer(&dst)) - aw := *(*[]uintptr)(unsafe.Pointer(&a)) - bw := *(*[]uintptr)(unsafe.Pointer(&b)) - for i := 0; i < w; i++ { - dw[i] = aw[i] ^ bw[i] - } - } - for i := n - n%wordSize; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - return n -} - -// safeXORBytes xors one by one. It works on all architectures, independent if -// it supports unaligned read/writes or not. -func safeXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - for i := 0; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - return n + return subtle.XORBytes(dst, a, b) } // ANDBytes ands the bytes in a and b. The destination is assumed to have enough diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 12f3fe24a6..1748029794 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -29,7 +29,7 @@ func TestXOR(t *testing.T) { d2 := make([]byte, 1023+alignD)[alignD:] XORBytes(d1, p, q) - safeXORBytes(d2, p, q) + naiveXOR(d2, p, q) if !bytes.Equal(d1, d2) { t.Error("not equal", d1, d2) } @@ -38,6 +38,18 @@ func TestXOR(t *testing.T) { } } +// naiveXOR xors bytes one by one. +func naiveXOR(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + // Tests that bitwise AND works for various alignments. func TestAND(t *testing.T) { for alignP := 0; alignP < 2; alignP++ { @@ -134,7 +146,7 @@ func benchmarkBaseXOR(b *testing.B, size int) { p, q := make([]byte, size), make([]byte, size) for i := 0; i < b.N; i++ { - safeXORBytes(p, p, q) + naiveXOR(p, p, q) } } diff --git a/common/size.go b/common/size.go index 097b6304a8..e7f504a082 100644 --- a/common/size.go +++ b/common/size.go @@ -26,13 +26,13 @@ type StorageSize float64 // String implements the stringer interface. func (s StorageSize) String() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2f TiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2f GiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2f MiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2f KiB", s/1024) } else { return fmt.Sprintf("%.2f B", s) @@ -42,13 +42,13 @@ func (s StorageSize) String() string { // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. func (s StorageSize) TerminalString() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2fTiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2fGiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2fMiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2fKiB", s/1024) } else { return fmt.Sprintf("%.2fB", s) diff --git a/common/types.go b/common/types.go index ebf5ca0bf6..4dedbf592a 100644 --- a/common/types.go +++ b/common/types.go @@ -71,6 +71,15 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } +// IsHexHash verifies whether a string can represent a valid hex-encoded +// Ethereum hash or not. +func IsHexHash(s string) bool { + if has0xPrefix(s) { + s = s[2:] + } + return len(s) == 2*HashLength && isHex(s) +} + // Cmp compares two hashes. func (h Hash) Cmp(other Hash) int { return bytes.Compare(h[:], other[:]) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index be46ebe906..43ba8c986e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -259,11 +259,11 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if !cancun { switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } } else { if header.ParentBeaconRoot == nil { @@ -366,46 +366,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea header.Root = state.IntermediateRoot(true) // Assemble the final block. - block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) - - // Create the block witness and attach to block. - // This step needs to happen as late as possible to catch all access events. - if chain.Config().IsVerkle(header.Number, header.Time) { - keys := state.AccessEvents().Keys() - - // Open the pre-tree to prove the pre-state against - parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1) - if parent == nil { - return nil, fmt.Errorf("nil parent header for block %d", header.Number) - } - preTrie, err := state.Database().OpenTrie(parent.Root) - if err != nil { - return nil, fmt.Errorf("error opening pre-state tree root: %w", err) - } - postTrie := state.GetTrie() - if postTrie == nil { - return nil, errors.New("post-state tree is not available") - } - vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := postTrie.(*trie.VerkleTrie) - - // The witness is only attached iff both parent and current block are - // using verkle tree. - if okpre && okpost { - if len(keys) > 0 { - verkleProof, stateDiff, err := vktPreTrie.Proof(vktPostTrie, keys) - if err != nil { - return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err) - } - block = block.WithWitness(&types.ExecutionWitness{ - StateDiff: stateDiff, - VerkleProof: verkleProof, - }) - } - } - } - - return block, nil + return types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)), nil } // Seal generates a new sealing request for the given input block and pushes diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 59d027225f..137d987a71 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -306,11 +306,11 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } // All basic checks passed, verify cascading fields return c.verifyCascadingFields(chain, header, parents) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index ecaaa8ef9f..d99363e434 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -280,11 +280,11 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } // Add some fake checks for tests if ethash.fakeDelay != nil { diff --git a/consensus/misc/eip1559/eip1559.go b/consensus/misc/eip1559/eip1559.go index a90bd744b2..9a4cd320a8 100644 --- a/consensus/misc/eip1559/eip1559.go +++ b/consensus/misc/eip1559/eip1559.go @@ -43,6 +43,10 @@ func VerifyEIP1559Header(config *params.ChainConfig, parent, header *types.Heade if header.BaseFee == nil { return errors.New("header is missing baseFee") } + // Verify the parent header is not malformed + if config.IsLondon(parent.Number) && parent.BaseFee == nil { + return errors.New("parent header is missing baseFee") + } // Verify the baseFee is correct based on the parent header. expectedBaseFee := CalcBaseFee(config, parent) if header.BaseFee.Cmp(expectedBaseFee) != 0 { diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 37af914212..ccc8321d47 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -53,9 +53,9 @@ func (bc *BlobConfig) blobPrice(excessBlobGas uint64) *big.Int { return new(big.Int).Mul(f, big.NewInt(params.BlobTxBlobGasPerBlob)) } -func latestBlobConfig(cfg *params.ChainConfig, time uint64) *BlobConfig { +func latestBlobConfig(cfg *params.ChainConfig, time uint64) (BlobConfig, error) { if cfg.BlobScheduleConfig == nil { - return nil + return BlobConfig{}, errors.New("no blob config") } var ( london = cfg.LondonBlock @@ -83,14 +83,14 @@ func latestBlobConfig(cfg *params.ChainConfig, time uint64) *BlobConfig { case cfg.IsCancun(london, time, 0) && s.Cancun != nil: bc = s.Cancun default: - return nil + return BlobConfig{}, errors.New("no blob config") } - return &BlobConfig{ + return BlobConfig{ Target: bc.Target, Max: bc.Max, UpdateFraction: bc.UpdateFraction, - } + }, nil } // VerifyEIP4844Header verifies the presence of the excessBlobGas field and that @@ -101,8 +101,8 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade panic("bad header pair") } - bcfg := latestBlobConfig(config, header.Time) - if bcfg == nil { + bcfg, err := latestBlobConfig(config, header.Time) + if err != nil { panic("called before EIP-4844 is active") } @@ -118,7 +118,7 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, bcfg.maxBlobGas()) } if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 { - return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) + return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", *header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header @@ -134,14 +134,14 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header, headTimestamp uint64) uint64 { // we can use 0 here because arbitrum doesn't support Blob transactions. isOsaka := config.IsOsaka(config.LondonBlock, headTimestamp, 0) - bcfg := latestBlobConfig(config, headTimestamp) - if bcfg == nil { - return 0 + bcfg, err := latestBlobConfig(config, headTimestamp) + if err != nil { + panic("calculating excess blob gas on nil blob config") } return calcExcessBlobGas(isOsaka, bcfg, parent) } -func calcExcessBlobGas(isOsaka bool, bcfg *BlobConfig, parent *types.Header) uint64 { +func calcExcessBlobGas(isOsaka bool, bcfg BlobConfig, parent *types.Header) uint64 { var parentExcessBlobGas, parentBlobGasUsed uint64 if parent.ExcessBlobGas != nil { parentExcessBlobGas = *parent.ExcessBlobGas @@ -180,8 +180,8 @@ func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { // Arbitrum does not support blob transactions, so we return 0. return big.NewInt(0) } - blobConfig := latestBlobConfig(config, header.Time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(config, header.Time) + if err != nil { panic("calculating blob fee on unsupported fork") } return blobConfig.blobBaseFee(*header.ExcessBlobGas) @@ -189,8 +189,8 @@ func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { // MaxBlobsPerBlock returns the max blobs per block for a block at the given timestamp. func MaxBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { - blobConfig := latestBlobConfig(cfg, time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { return 0 } return blobConfig.Max @@ -204,8 +204,8 @@ func MaxBlobGasPerBlock(cfg *params.ChainConfig, time uint64) uint64 { // LatestMaxBlobsPerBlock returns the latest max blobs per block defined by the // configuration, regardless of the currently active fork. func LatestMaxBlobsPerBlock(cfg *params.ChainConfig) int { - bcfg := latestBlobConfig(cfg, math.MaxUint64) - if bcfg == nil { + bcfg, err := latestBlobConfig(cfg, math.MaxUint64) + if err != nil { return 0 } return bcfg.Max @@ -226,6 +226,15 @@ func CalcBlobFeeWithConfig(bc *params.BlobConfig, excessBlobGas *uint64) *big.In return gethBc.blobBaseFee(*excessBlobGas) } +// TargetBlobsPerBlock returns the target blobs per block for a block at the given timestamp. +func TargetBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { + return 0 + } + return blobConfig.Target +} + // fakeExponential approximates factor * e ** (numerator / denominator) using // Taylor expansion. func fakeExponential(factor, numerator, denominator *big.Int) *big.Int { diff --git a/console/prompt/prompter.go b/console/prompt/prompter.go index 2a20b6906a..5a0a89e76a 100644 --- a/console/prompt/prompter.go +++ b/console/prompt/prompter.go @@ -142,7 +142,7 @@ func (p *terminalPrompter) PromptPassword(prompt string) (passwd string, err err // PromptConfirm displays the given prompt to the user and requests a boolean // choice to be made, returning that choice. func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) { - input, err := p.Prompt(prompt + " [y/n] ") + input, err := p.PromptInput(prompt + " [y/n] ") if len(input) > 0 && strings.EqualFold(input[:1], "y") { return true, nil } diff --git a/core/bintrie_witness_test.go b/core/bintrie_witness_test.go new file mode 100644 index 0000000000..0c8a038776 --- /dev/null +++ b/core/bintrie_witness_test.go @@ -0,0 +1,237 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "encoding/binary" + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/triedb" +) + +var ( + testVerkleChainConfig = ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + ShanghaiTime: u64(0), + VerkleTime: u64(0), + TerminalTotalDifficulty: common.Big0, + EnableVerkleAtGenesis: true, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Verkle: params.DefaultPragueBlobConfig, + }, + } +) + +func TestProcessVerkle(t *testing.T) { + var ( + code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) + intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) + // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness + // will not contain that copied data. + // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 + codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) + intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) + signer = types.LatestSigner(testVerkleChainConfig) + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain + coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") + gspec = &Genesis{ + Config: testVerkleChainConfig, + Alloc: GenesisAlloc{ + coinbase: { + Balance: big.NewInt(1000000000000000000), // 1 ether + Nonce: 0, + }, + params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, + params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, + params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, + params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, + }, + } + ) + // Verkle trees use the snapshot, which must be enabled before the + // data is saved into the tree+database. + // genesis := gspec.MustCommit(bcdb, triedb) + options := DefaultConfig().WithStateScheme(rawdb.PathScheme) + options.SnapshotLimit = 0 + blockchain, _ := NewBlockChain(bcdb, nil, gspec, beacon.New(ethash.NewFaker()), options) + defer blockchain.Stop() + + txCost1 := params.TxGas + txCost2 := params.TxGas + contractCreationCost := intrinsicContractCreationGas + + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* creation with value */ + 739 /* execution costs */ + codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (tx) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at pc=0x20) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ + params.WitnessChunkReadCost + /* SLOAD in constructor */ + params.WitnessChunkWriteCost + /* SSTORE in constructor */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at PC=0x121) */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ + params.WitnessChunkReadCost + /* SLOAD in constructor */ + params.WitnessChunkWriteCost + /* SSTORE in constructor */ + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ + 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ + uint64(4844) /* execution costs */ + blockGasUsagesExpected := []uint64{ + txCost1*2 + txCost2, + txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, + } + _, chain, _ := GenerateChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + gen.SetPoS() + + // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) + tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + + // Add two contract creations in block #2 + if i == 1 { + tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 6, + Value: big.NewInt(16), + Gas: 3000000, + GasPrice: big.NewInt(875000000), + Data: code, + }) + gen.AddTx(tx) + + tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 7, + Value: big.NewInt(0), + Gas: 3000000, + GasPrice: big.NewInt(875000000), + Data: codeWithExtCodeCopy, + }) + gen.AddTx(tx) + } + }) + + for i, b := range chain { + fmt.Printf("%d %x\n", i, b.Root()) + } + endnum, err := blockchain.InsertChain(chain) + if err != nil { + t.Fatalf("block %d imported with error: %v", endnum, err) + } + + for i := range 2 { + b := blockchain.GetBlockByNumber(uint64(i) + 1) + if b == nil { + t.Fatalf("expected block %d to be present in chain", i+1) + } + if b.Hash() != chain[i].Hash() { + t.Fatalf("block #%d not found at expected height", b.NumberU64()) + } + if b.GasUsed() != blockGasUsagesExpected[i] { + t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed()) + } + } +} + +func TestProcessParentBlockHash(t *testing.T) { + // This test uses blocks where, + // block 1 parent hash is 0x0100.... + // block 2 parent hash is 0x0200.... + // etc + checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { + statedb.SetNonce(params.HistoryStorageAddress, 1, tracing.NonceChangeUnspecified) + statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode, tracing.CodeChangeUnspecified) + // Process n blocks, from 1 .. num + var num = 2 + for i := 1; i <= num; i++ { + header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} + chainConfig := params.MergedTestChainConfig + if isVerkle { + chainConfig = testVerkleChainConfig + } + vmContext := NewEVMBlockContext(header, nil, new(common.Address)) + evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) + ProcessParentBlockHash(header.ParentHash, evm) + } + // Read block hashes for block 0 .. num-1 + for i := 0; i < num; i++ { + have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} + if have != want { + t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) + } + } + } + t.Run("MPT", func(t *testing.T) { + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + checkBlockHashes(statedb, false) + }) + t.Run("Verkle", func(t *testing.T) { + db := rawdb.NewMemoryDatabase() + cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) + cacheConfig.SnapshotLimit = 0 + triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) + statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) + checkBlockHashes(statedb, true) + }) +} + +// getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' +func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { + ringIndex := number % params.HistoryServeWindow + var key common.Hash + binary.BigEndian.PutUint64(key[24:], ringIndex) + if isVerkle { + return statedb.GetState(params.HistoryStorageAddress, key) + } + return statedb.GetState(params.HistoryStorageAddress, key) +} diff --git a/core/block_validator_test.go b/core/block_validator_test.go index ce026d56f7..f40d503845 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -17,6 +17,7 @@ package core import ( + "context" "math/big" "testing" "time" @@ -210,7 +211,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Fatalf("post-block %d: unexpected result returned: %v", i, result) case <-time.After(25 * time.Millisecond): } - chain.InsertBlockWithoutSetHead(postBlocks[i], false) + chain.InsertBlockWithoutSetHead(context.Background(), postBlocks[i], false) } // Verify the blocks with pre-merge blocks and post-merge blocks diff --git a/core/blockchain.go b/core/blockchain.go index a5439b82f6..4830f3f2be 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,6 +18,7 @@ package core import ( + "context" "errors" "fmt" "io" @@ -48,6 +49,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/syncx" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -76,6 +78,8 @@ var ( storageReadTimer = metrics.NewRegisteredResettingTimer("chain/storage/reads", nil) storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) + codeReadTimer = metrics.NewRegisteredResettingTimer("chain/code/reads", nil) + codeReadBytesTimer = metrics.NewRegisteredResettingTimer("chain/code/readbytes", nil) accountCacheHitMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/hit", nil) accountCacheMissMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/miss", nil) @@ -89,6 +93,7 @@ var ( accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil) storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil) + codeReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/code/single/reads", nil) snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) @@ -181,6 +186,17 @@ type BlockChainConfig struct { // If set to 0, all state histories across the entire chain will be retained; StateHistory uint64 + // Number of blocks from the chain head for which trienode histories are retained. + // If set to 0, all trienode histories across the entire chain will be retained; + // If set to -1, no trienode history will be retained; + TrienodeHistory int64 + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 + // State snapshot related options SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory SnapshotNoBuild bool // Whether the background generation is allowed @@ -215,6 +231,11 @@ type BlockChainConfig struct { // StateSizeTracking indicates whether the state size tracking is enabled. StateSizeTracking bool + + // SlowBlockThreshold is the block execution time threshold beyond which + // detailed statistics will be logged. Negative value means disabled (default), + // zero logs all blocks, positive value filters blocks by execution time. + SlowBlockThreshold time.Duration } // DefaultConfig returns the default config. @@ -288,18 +309,23 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config { } if cfg.StateScheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{ - StateHistory: cfg.StateHistory, - EnableStateIndexing: cfg.ArchiveMode, - TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024, - StateCleanSize: cfg.SnapshotLimit * 1024 * 1024, - JournalDirectory: cfg.TrieJournalDirectory, - MaxDiffLayers: cfg.MaxDiffLayers, - + TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024, + StateCleanSize: cfg.SnapshotLimit * 1024 * 1024, // TODO(rjl493456442): The write buffer represents the memory limit used // for flushing both trie data and state data to disk. The config name // should be updated to eliminate the confusion. - WriteBufferSize: cfg.TrieDirtyLimit * 1024 * 1024, - NoAsyncFlush: cfg.TrieNoAsyncFlush, + WriteBufferSize: cfg.TrieDirtyLimit * 1024 * 1024, + JournalDirectory: cfg.TrieJournalDirectory, + MaxDiffLayers: cfg.MaxDiffLayers, + + // Historical state configurations + StateHistory: cfg.StateHistory, + TrienodeHistory: cfg.TrienodeHistory, + EnableStateIndexing: cfg.ArchiveMode, + FullValueCheckpoint: cfg.NodeFullValueCheckpoint, + + // Testing configurations + NoAsyncFlush: cfg.TrieNoAsyncFlush, } } return config @@ -346,6 +372,7 @@ type BlockChain struct { chainHeadFeed event.Feed logsFeed event.Feed blockProcFeed event.Feed + newPayloadFeed event.Feed // Feed for engine API newPayload events blockProcCounter int32 scope event.SubscriptionScope genesisBlock *types.Block @@ -378,7 +405,8 @@ type BlockChain struct { logger *tracing.Hooks stateSizer *state.SizeTracker // State size tracking - lastForkReadyAlert time.Time // Last time there was a fork readiness print out + lastForkReadyAlert time.Time // Last time there was a fork readiness print out + slowBlockThreshold time.Duration // Block execution time threshold beyond which detailed statistics will be logged // Arbitrum: numberOfBlocksToSkipStateSaving uint32 @@ -419,7 +447,7 @@ func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, genesis * // yet. The corresponding chain config will be returned, either from the // provided genesis or from the locally stored configuration if the genesis // has already been initialized. - chainConfig, genesisHash, compatErr, err = SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides) + chainConfig, genesisHash, compatErr, err = SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides, cfg.VmConfig.Tracer) if err != nil { return nil, err } @@ -433,19 +461,20 @@ func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, genesis * log.Info("") bc := &BlockChain{ - chainConfig: chainConfig, - cfg: cfg, - db: db, - triedb: triedb, - triegc: prque.New[int64, trieGcEntry](nil), - chainmu: syncx.NewClosableMutex(), - bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), - bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), - receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), - blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - engine: engine, - logger: cfg.VmConfig.Tracer, + chainConfig: chainConfig, + cfg: cfg, + db: db, + triedb: triedb, + triegc: prque.New[int64, trieGcEntry](nil), + chainmu: syncx.NewClosableMutex(), + bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), + bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), + receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), + blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), + txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), + engine: engine, + logger: cfg.VmConfig.Tracer, + slowBlockThreshold: cfg.SlowBlockThreshold, } bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) if err != nil { @@ -808,21 +837,7 @@ func (bc *BlockChain) SetHead(head uint64) error { if _, _, err := bc.setHeadBeyondRoot(head, 0, common.Hash{}, false, 0); err != nil { return err } - // Send chain head event to update the transaction pool - header := bc.CurrentBlock() - if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // In a pruned node the genesis block will not exist in the freezer. - // It should not happen that we set head to any other pruned block. - if header.Number.Uint64() > 0 { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) - } - } - bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) - return nil + return bc.sendChainHeadEvent() } // SetHeadWithTimestamp rewinds the local chain to a new head that has at max @@ -833,7 +848,12 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { if _, _, err := bc.setHeadBeyondRoot(0, timestamp, common.Hash{}, false, 0); err != nil { return err } - // Send chain head event to update the transaction pool + return bc.sendChainHeadEvent() +} + +// sendChainHeadEvent notifies all subscribers about the new chain head, +// checking first that the current block is actually available. +func (bc *BlockChain) sendChainHeadEvent() error { header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { // In a pruned node the genesis block will not exist in the freezer. @@ -1046,7 +1066,8 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ // Recover if the target state if it's not available yet. if !bc.HasState(head.Root) { if err := bc.triedb.Recover(head.Root); err != nil { - log.Crit("Failed to rollback state", "err", err) + log.Error("Failed to rollback state, resetting to genesis", "err", err) + return bc.genesisBlock.Header(), rootNumber } } log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) @@ -1200,11 +1221,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.txLookupCache.Purge() // Clear safe block, finalized block if needed - if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() { + headBlock := bc.CurrentBlock() + if safe := bc.CurrentSafeBlock(); safe != nil && headBlock.Number.Uint64() < safe.Number.Uint64() { log.Warn("SetHead invalidated safe block") bc.SetSafe(nil) } - if finalized := bc.CurrentFinalBlock(); finalized != nil && head < finalized.Number.Uint64() { + if finalized := bc.CurrentFinalBlock(); finalized != nil && headBlock.Number.Uint64() < finalized.Number.Uint64() { log.Error("SetHead invalidated finalized block") bc.SetFinalized(nil) } @@ -1212,14 +1234,48 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha return blockNumber, rootFound, bc.loadLastState() } -// SnapSyncCommitHead sets the current head block to the one defined by the hash -// irrelevant what the chain contents were prior. -func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { +// SnapSyncStart disables the underlying databases (such as the trie DB and the +// optional state snapshot) to prevent potential concurrent mutations between +// snap sync and other chain operations. +func (bc *BlockChain) SnapSyncStart() error { + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + + // Snap sync will directly modify the persistent state, making the entire + // trie database unusable until the state is fully synced. To prevent any + // subsequent state reads, explicitly disable the trie database and state + // syncer is responsible to address and correct any state missing. + if bc.TrieDB().Scheme() == rawdb.PathScheme { + if err := bc.TrieDB().Disable(); err != nil { + return err + } + } + // Snap sync uses the snapshot namespace to store potentially flaky data until + // sync completely heals and finishes. Pause snapshot maintenance in the mean- + // time to prevent access. + if snapshots := bc.Snapshots(); snapshots != nil { // Only nil in tests + snapshots.Disable() + } + return nil +} + +// SnapSyncComplete sets the current head block to the block identified by the +// given hash, regardless of the chain contents prior to snap sync. It is +// invoked once snap sync completes and assumes that SnapSyncStart was called +// previously. +func (bc *BlockChain) SnapSyncComplete(hash common.Hash) error { // Make sure that both the block as well at its state trie exists block := bc.GetBlockByHash(hash) if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } + if !bc.chainmu.TryLock() { + return errChainStopped + } + defer bc.chainmu.Unlock() + // Reset the trie database with the fresh snap synced state. root := block.Root() if bc.triedb.Scheme() == rawdb.PathScheme { @@ -1230,19 +1286,16 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if !bc.HasState(root) { return fmt.Errorf("non existent state [%x..]", root[:4]) } - // If all checks out, manually set the head block. - if !bc.chainmu.TryLock() { - return errChainStopped - } - bc.currentBlock.Store(block.Header()) - headBlockGauge.Update(int64(block.NumberU64())) - bc.chainmu.Unlock() - // Destroy any existing state snapshot and regenerate it in the background, // also resuming the normal maintenance of any previously paused snapshot. if bc.snaps != nil { bc.snaps.Rebuild(root) } + + // If all checks out, manually set the head block. + bc.currentBlock.Store(block.Header()) + headBlockGauge.Update(int64(block.NumberU64())) + log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil } @@ -1721,21 +1774,47 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // // Note all the components of block(hash->number map, header, body, receipts) // should be written atomically. BlockBatch is used for containing all components. - blockBatch := bc.db.NewBatch() - rawdb.WriteBlock(blockBatch, block) - rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts) - rawdb.WritePreimages(blockBatch, statedb.Preimages()) - if err := blockBatch.Write(); err != nil { + var ( + batch = bc.db.NewBatch() + start = time.Now() + ) + rawdb.WriteBlock(batch, block) + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + rawdb.WritePreimages(batch, statedb.Preimages()) + if err := batch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) } - // Commit all cached state changes into underlying memory database. - root, stateUpdate, err := statedb.CommitWithUpdate(block.NumberU64(), bc.chainConfig.IsEIP158(block.Number()), bc.chainConfig.IsCancun(block.Number(), block.Time(), types.DeserializeHeaderExtraInformation(block.Header()).ArbOSFormatVersion)) - if err != nil { - return err - } - // Emit the state update to the state sizestats if it's active - if bc.stateSizer != nil { - bc.stateSizer.Notify(stateUpdate) + log.Debug("Committed block data", "size", common.StorageSize(batch.ValueSize()), "elapsed", common.PrettyDuration(time.Since(start))) + + var ( + err error + root common.Hash + isEIP158 = bc.chainConfig.IsEIP158(block.Number()) + isCancun = bc.chainConfig.IsCancun(block.Number(), block.Time(), types.DeserializeHeaderExtraInformation(block.Header()).ArbOSFormatVersion) + hasStateHook = bc.logger != nil && bc.logger.OnStateUpdate != nil + hasStateSizer = bc.stateSizer != nil + ) + if hasStateHook || hasStateSizer { + r, update, err := statedb.CommitWithUpdate(block.NumberU64(), isEIP158, isCancun) + if err != nil { + return err + } + if hasStateHook { + trUpdate, err := update.ToTracingUpdate() + if err != nil { + return err + } + bc.logger.OnStateUpdate(trUpdate) + } + if hasStateSizer { + bc.stateSizer.Notify(update) + } + root = r + } else { + root, err = statedb.Commit(block.NumberU64(), isEIP158, isCancun) + if err != nil { + return err + } } // If node is running in path mode, skip explicit gc operation // which is unnecessary in this mode. @@ -1908,7 +1987,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } defer bc.chainmu.Unlock() - _, n, err := bc.insertChain(chain, true, false) // No witness collection for mass inserts (would get super large) + _, n, err := bc.insertChain(context.Background(), chain, true, false) // No witness collection for mass inserts (would get super large) return n, err } @@ -1920,7 +1999,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertChain(ctx context.Context, chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { // If the chain is terminating, don't even bother starting up. if bc.insertStopped() { return nil, 0, nil @@ -2003,11 +2082,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness if setHead { // First block is pruned, insert as sidechain and reorg only if TD grows enough log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it, makeWitness) + return bc.insertSideChain(ctx, block, it, makeWitness) } else { // We're post-merge and the parent is pruned, try to recover the parent state log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) - _, err := bc.recoverAncestors(block, makeWitness) + _, err := bc.recoverAncestors(ctx, block, makeWitness) return nil, it.index, err } // Some other error(except ErrKnownBlock) occurred, abort. @@ -2015,7 +2094,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness // still need re-execution to generate snapshots that are missing case err != nil && !errors.Is(err, ErrKnownBlock): stats.ignored += len(it.chain) - bc.reportBlock(block, nil, err) + bc.reportBadBlock(block, nil, err) return nil, it.index, err } // Track the singleton witness from this chain insertion (if any) @@ -2079,10 +2158,18 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness } // The traced section of block import. start := time.Now() - res, err := bc.ProcessBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1) + res, err := bc.ProcessBlock(ctx, parent.Root, block, setHead, makeWitness && len(chain) == 1) if err != nil { return nil, it.index, err } + res.stats.reportMetrics() + + // Log slow block only if a single block is inserted (usually after the + // initial sync) to not overwhelm the users. + if len(chain) == 1 { + res.stats.logSlow(block, bc.slowBlockThreshold) + } + // Report the import stats before returning the various results stats.processed++ stats.usedGas += res.usedGas @@ -2143,15 +2230,20 @@ type blockProcessingResult struct { procTime time.Duration status WriteStatus witness *stateless.Witness + stats *ExecuteStats } func (bpr *blockProcessingResult) Witness() *stateless.Witness { return bpr.witness } +func (bpr *blockProcessingResult) Stats() *ExecuteStats { + return bpr.stats +} + // ProcessBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. -func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) { +func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (result *blockProcessingResult, blockEndErr error) { var ( err error startTime = time.Now() @@ -2185,18 +2277,11 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s } // Upload the statistics of reader at the end defer func() { - stats := prefetch.GetStats() - accountCacheHitPrefetchMeter.Mark(stats.AccountHit) - accountCacheMissPrefetchMeter.Mark(stats.AccountMiss) - storageCacheHitPrefetchMeter.Mark(stats.StorageHit) - storageCacheMissPrefetchMeter.Mark(stats.StorageMiss) - stats = process.GetStats() - accountCacheHitMeter.Mark(stats.AccountHit) - accountCacheMissMeter.Mark(stats.AccountMiss) - storageCacheHitMeter.Mark(stats.StorageHit) - storageCacheMissMeter.Mark(stats.StorageMiss) + if result != nil { + result.stats.StatePrefetchCacheStats = prefetch.GetStats() + result.stats.StateReadCacheStats = process.GetStats() + } }() - go func(start time.Time, throwaway *state.StateDB, block *types.Block) { // Disable tracing for prefetcher executions. vmCfg := bc.cfg.VmConfig @@ -2249,16 +2334,21 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // Process block using the parent state as reference point pstart := time.Now() - res, err := bc.processor.Process(block, statedb, bc.cfg.VmConfig) + pctx, _, spanEnd := telemetry.StartSpan(ctx, "bc.processor.Process") + res, err := bc.processor.Process(pctx, block, statedb, bc.cfg.VmConfig) + spanEnd(&err) if err != nil { - bc.reportBlock(block, res, err) + bc.reportBadBlock(block, res, err) return nil, err } ptime := time.Since(pstart) vstart := time.Now() - if err := bc.validator.ValidateState(block, statedb, res, false); err != nil { - bc.reportBlock(block, res, err) + _, _, spanEnd = telemetry.StartSpan(ctx, "bc.validator.ValidateState") + err = bc.validator.ValidateState(block, statedb, res, false) + spanEnd(&err) + if err != nil { + bc.reportBadBlock(block, res, err) return nil, err } vtime := time.Since(vstart) @@ -2280,7 +2370,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s task := types.NewBlockWithHeader(context).WithBody(*block.Body()) // Run the stateless self-cross-validation - crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, bc.cfg.VmConfig, task, witness) + crossStateRoot, crossReceiptRoot, err := ExecuteStateless(ctx, bc.chainConfig, bc.cfg.VmConfig, task, witness) if err != nil { return nil, fmt.Errorf("stateless self-validation failed: %v", err) } @@ -2292,26 +2382,34 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s } } - xvtime := time.Since(xvstart) - proctime := time.Since(startTime) // processing + validation + cross validation - + var ( + xvtime = time.Since(xvstart) + proctime = time.Since(startTime) // processing + validation + cross validation + stats = &ExecuteStats{} + ) // Update the metrics touched during block processing and validation - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) - if statedb.AccountLoaded != 0 { - accountReadSingleTimer.Update(statedb.AccountReads / time.Duration(statedb.AccountLoaded)) - } - if statedb.StorageLoaded != 0 { - storageReadSingleTimer.Update(statedb.StorageReads / time.Duration(statedb.StorageLoaded)) - } - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) - triehash := statedb.AccountHashes // The time spent on tries hashing - trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on tries update - blockExecutionTimer.Update(ptime - (statedb.AccountReads + statedb.StorageReads)) // The time spent on EVM processing - blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation - blockCrossValidationTimer.Update(xvtime) // The time spent on stateless cross validation + stats.AccountReads = statedb.AccountReads // Account reads are complete(in processing) + stats.StorageReads = statedb.StorageReads // Storage reads are complete(in processing) + stats.AccountUpdates = statedb.AccountUpdates // Account updates are complete(in validation) + stats.StorageUpdates = statedb.StorageUpdates // Storage updates are complete(in validation) + stats.AccountHashes = statedb.AccountHashes // Account hashes are complete(in validation) + stats.CodeReads = statedb.CodeReads + + stats.AccountLoaded = statedb.AccountLoaded + stats.AccountUpdated = statedb.AccountUpdated + stats.AccountDeleted = statedb.AccountDeleted + stats.StorageLoaded = statedb.StorageLoaded + stats.StorageUpdated = int(statedb.StorageUpdated.Load()) + stats.StorageDeleted = int(statedb.StorageDeleted.Load()) + + stats.CodeLoaded = statedb.CodeLoaded + stats.CodeLoadBytes = statedb.CodeLoadBytes + stats.CodeUpdated = statedb.CodeUpdated + stats.CodeUpdateBytes = statedb.CodeUpdateBytes + + stats.Execution = ptime - (statedb.AccountReads + statedb.StorageReads + statedb.CodeReads) // The time spent on EVM processing + stats.Validation = vtime - (statedb.AccountHashes + statedb.AccountUpdates + statedb.StorageUpdates) // The time spent on block validation + stats.CrossValidation = xvtime // The time spent on stateless cross validation // Write the block to the chain and get the status. var ( @@ -2333,27 +2431,26 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s } // Update the metrics touched during block commit - accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them - storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them - snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them - triedbCommitTimer.Update(statedb.TrieDBCommits) // Trie database commits are complete, we can mark them + stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them + stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them + stats.SnapshotCommit = statedb.SnapshotCommits // Snapshot commits are complete, we can mark them + stats.TrieDBCommit = statedb.TrieDBCommits // Trie database commits are complete, we can mark them + stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits - blockWriteTimer.Update(time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits) elapsed := time.Since(startTime) + 1 // prevent zero division - blockInsertTimer.Update(elapsed) + stats.TotalTime = elapsed + stats.MgasPerSecond = float64(res.GasUsed) * 1000 / float64(elapsed) + if bc.logger != nil && bc.logger.OnBlockEndMetrics != nil { bc.logger.OnBlockEndMetrics(block.NumberU64(), elapsed) } - // TODO(rjl493456442) generalize the ResettingTimer - mgasps := float64(res.GasUsed) * 1000 / float64(elapsed) - chainMgaspsMeter.Update(time.Duration(mgasps)) - return &blockProcessingResult{ usedGas: res.GasUsed, procTime: proctime, status: status, witness: witness, + stats: stats, }, nil } @@ -2364,7 +2461,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. // insertSideChain is only used pre-merge. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertSideChain(ctx context.Context, block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { var current = bc.CurrentBlock() // The first sidechain block error is already verified to be ErrPrunedAncestor. @@ -2445,7 +2542,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, _, err := bc.insertChain(blocks, true, false); err != nil { + if _, _, err := bc.insertChain(ctx, blocks, true, false); err != nil { return nil, 0, err } blocks, memory = blocks[:0], 0 @@ -2459,7 +2556,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, true, makeWitness) + return bc.insertChain(ctx, blocks, true, makeWitness) } return nil, 0, nil } @@ -2468,7 +2565,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // all the ancestor blocks since that. // recoverAncestors is only used post-merge. // We return the hash of the latest block that we could correctly validate. -func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (common.Hash, error) { +func (bc *BlockChain) recoverAncestors(ctx context.Context, block *types.Block, makeWitness bool) (common.Hash, error) { // Gather all the sidechain hashes (full blocks may be memory heavy) var ( hashes []common.Hash @@ -2508,7 +2605,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (co } else { b = bc.GetBlock(hashes[i], numbers[i]) } - if _, _, err := bc.insertChain(types.Blocks{b}, false, makeWitness && i == 0); err != nil { + if _, _, err := bc.insertChain(ctx, types.Blocks{b}, false, makeWitness && i == 0); err != nil { return b.ParentHash(), err } } @@ -2743,14 +2840,16 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // The key difference between the InsertChain is it won't do the canonical chain // updating. It relies on the additional SetCanonical call to finalize the entire // procedure. -func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block, makeWitness bool) (*stateless.Witness, error) { +func (bc *BlockChain) InsertBlockWithoutSetHead(ctx context.Context, block *types.Block, makeWitness bool) (witness *stateless.Witness, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.blockchain.InsertBlockWithoutSetHead") + defer spanEnd(&err) if !bc.chainmu.TryLock() { return nil, errChainStopped } defer bc.chainmu.Unlock() - witness, _, err := bc.insertChain(types.Blocks{block}, false, makeWitness) - return witness, err + witness, _, err = bc.insertChain(ctx, types.Blocks{block}, false, makeWitness) + return } // SetCanonical rewinds the chain to set the new head block as the specified @@ -2764,7 +2863,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { // Re-execute the reorged chain in case the head state is missing. if !bc.HasState(head.Root()) { - if latestValidHash, err := bc.recoverAncestors(head, false); err != nil { + if latestValidHash, err := bc.recoverAncestors(context.Background(), head, false); err != nil { return latestValidHash, err } log.Info("Recovered head state", "number", head.Number(), "hash", head.Hash()) @@ -2846,8 +2945,8 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { return false } -// reportBlock logs a bad block error. -func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err error) { +// reportBadBlock logs a bad block error. +func (bc *BlockChain) reportBadBlock(block *types.Block, res *ProcessResult, err error) { var receipts types.Receipts if res != nil { receipts = res.Receipts diff --git a/core/blockchain_arbitrum.go b/core/blockchain_arbitrum.go index 0bb075c239..25f45146ae 100644 --- a/core/blockchain_arbitrum.go +++ b/core/blockchain_arbitrum.go @@ -18,6 +18,7 @@ package core import ( + "context" "fmt" "math/rand/v2" "time" @@ -106,11 +107,11 @@ func (bc *BlockChain) ClipToPostNitroGenesis(blockNum rpc.BlockNumber) (rpc.Bloc return blockNum, currentBlock } -func (bc *BlockChain) RecoverState(block *types.Block) error { +func (bc *BlockChain) RecoverState(ctx context.Context, block *types.Block) error { if bc.HasState(block.Root()) { return nil } log.Warn("recovering block state", "num", block.Number(), "hash", block.Hash(), "root", block.Root()) - _, err := bc.recoverAncestors(block, false) + _, err := bc.recoverAncestors(ctx, block, false) return err } diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index ac6a156d3e..07a250a1bb 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -131,28 +131,6 @@ func (it *insertIterator) next() (*types.Block, error) { return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index]) } -// peek returns the next block in the iterator, along with any potential validation -// error for that block, but does **not** advance the iterator. -// -// Both header and body validation errors (nil too) is cached into the iterator -// to avoid duplicating work on the following next() call. -// nolint:unused -func (it *insertIterator) peek() (*types.Block, error) { - // If we reached the end of the chain, abort - if it.index+1 >= len(it.chain) { - return nil, nil - } - // Wait for verification result if not yet done - if len(it.errors) <= it.index+1 { - it.errors = append(it.errors, <-it.results) - } - if it.errors[it.index+1] != nil { - return it.chain[it.index+1], it.errors[it.index+1] - } - // Block header valid, ignore body validation since we don't have a parent anyway - return it.chain[it.index+1], nil -} - // previous returns the previous header that was being processed, or nil. func (it *insertIterator) previous() *types.Header { if it.index < 1 { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 8160e5364c..af2aeccd5a 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -530,3 +530,13 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription { return bc.scope.Track(bc.blockProcFeed.Subscribe(ch)) } + +// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent. +func (bc *BlockChain) SubscribeNewPayloadEvent(ch chan<- NewPayloadEvent) event.Subscription { + return bc.scope.Track(bc.newPayloadFeed.Subscribe(ch)) +} + +// SendNewPayloadEvent sends a NewPayloadEvent to subscribers. +func (bc *BlockChain) SendNewPayloadEvent(ev NewPayloadEvent) { + bc.newPayloadFeed.Send(ev) +} diff --git a/core/blockchain_stats.go b/core/blockchain_stats.go new file mode 100644 index 0000000000..adc66266c4 --- /dev/null +++ b/core/blockchain_stats.go @@ -0,0 +1,265 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "encoding/json" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// ExecuteStats includes all the statistics of a block execution in details. +type ExecuteStats struct { + // State read times + AccountReads time.Duration // Time spent on the account reads + StorageReads time.Duration // Time spent on the storage reads + AccountHashes time.Duration // Time spent on the account trie hash + AccountUpdates time.Duration // Time spent on the account trie update + AccountCommits time.Duration // Time spent on the account trie commit + StorageUpdates time.Duration // Time spent on the storage trie update + StorageCommits time.Duration // Time spent on the storage trie commit + CodeReads time.Duration // Time spent on the contract code read + + AccountLoaded int // Number of accounts loaded + AccountUpdated int // Number of accounts updated + AccountDeleted int // Number of accounts deleted + StorageLoaded int // Number of storage slots loaded + StorageUpdated int // Number of storage slots updated + StorageDeleted int // Number of storage slots deleted + CodeLoaded int // Number of contract code loaded + CodeLoadBytes int // Number of bytes read from contract code + CodeUpdated int // Number of contract code written (CREATE/CREATE2 + EIP-7702) + CodeUpdateBytes int // Total bytes of code written + + Execution time.Duration // Time spent on the EVM execution + Validation time.Duration // Time spent on the block validation + CrossValidation time.Duration // Optional, time spent on the block cross validation + SnapshotCommit time.Duration // Time spent on snapshot commit + TrieDBCommit time.Duration // Time spent on database commit + BlockWrite time.Duration // Time spent on block write + TotalTime time.Duration // The total time spent on block execution + MgasPerSecond float64 // The million gas processed per second + + // Cache hit rates + StateReadCacheStats state.ReaderStats + StatePrefetchCacheStats state.ReaderStats +} + +// reportMetrics uploads execution statistics to the metrics system. +func (s *ExecuteStats) reportMetrics() { + if s.AccountLoaded != 0 { + accountReadTimer.Update(s.AccountReads) + accountReadSingleTimer.Update(s.AccountReads / time.Duration(s.AccountLoaded)) + } + if s.StorageLoaded != 0 { + storageReadTimer.Update(s.StorageReads) + storageReadSingleTimer.Update(s.StorageReads / time.Duration(s.StorageLoaded)) + } + if s.CodeLoaded != 0 { + codeReadTimer.Update(s.CodeReads) + codeReadSingleTimer.Update(s.CodeReads / time.Duration(s.CodeLoaded)) + codeReadBytesTimer.Update(time.Duration(s.CodeLoadBytes)) + } + accountUpdateTimer.Update(s.AccountUpdates) // Account updates are complete(in validation) + storageUpdateTimer.Update(s.StorageUpdates) // Storage updates are complete(in validation) + accountHashTimer.Update(s.AccountHashes) // Account hashes are complete(in validation) + accountCommitTimer.Update(s.AccountCommits) // Account commits are complete, we can mark them + storageCommitTimer.Update(s.StorageCommits) // Storage commits are complete, we can mark them + + blockExecutionTimer.Update(s.Execution) // The time spent on EVM processing + blockValidationTimer.Update(s.Validation) // The time spent on block validation + blockCrossValidationTimer.Update(s.CrossValidation) // The time spent on stateless cross validation + snapshotCommitTimer.Update(s.SnapshotCommit) // Snapshot commits are complete, we can mark them + triedbCommitTimer.Update(s.TrieDBCommit) // Trie database commits are complete, we can mark them + blockWriteTimer.Update(s.BlockWrite) // The time spent on block write + blockInsertTimer.Update(s.TotalTime) // The total time spent on block execution + chainMgaspsMeter.Update(time.Duration(s.MgasPerSecond)) // TODO(rjl493456442) generalize the ResettingTimer + + // Cache hit rates + accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheHit) + accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheMiss) + storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheHit) + storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheMiss) + + accountCacheHitMeter.Mark(s.StateReadCacheStats.AccountCacheHit) + accountCacheMissMeter.Mark(s.StateReadCacheStats.AccountCacheMiss) + storageCacheHitMeter.Mark(s.StateReadCacheStats.StorageCacheHit) + storageCacheMissMeter.Mark(s.StateReadCacheStats.StorageCacheMiss) +} + +// slowBlockLog represents the JSON structure for slow block logging. +// This format is designed for cross-client compatibility with other +// Ethereum execution clients (reth, Besu, Nethermind). +type slowBlockLog struct { + Level string `json:"level"` + Msg string `json:"msg"` + Block slowBlockInfo `json:"block"` + Timing slowBlockTime `json:"timing"` + Throughput slowBlockThru `json:"throughput"` + StateReads slowBlockReads `json:"state_reads"` + StateWrites slowBlockWrites `json:"state_writes"` + Cache slowBlockCache `json:"cache"` +} + +type slowBlockInfo struct { + Number uint64 `json:"number"` + Hash common.Hash `json:"hash"` + GasUsed uint64 `json:"gas_used"` + TxCount int `json:"tx_count"` +} + +type slowBlockTime struct { + ExecutionMs float64 `json:"execution_ms"` + StateReadMs float64 `json:"state_read_ms"` + StateHashMs float64 `json:"state_hash_ms"` + CommitMs float64 `json:"commit_ms"` + TotalMs float64 `json:"total_ms"` +} + +type slowBlockThru struct { + MgasPerSec float64 `json:"mgas_per_sec"` +} + +type slowBlockReads struct { + Accounts int `json:"accounts"` + StorageSlots int `json:"storage_slots"` + Code int `json:"code"` + CodeBytes int `json:"code_bytes"` +} + +type slowBlockWrites struct { + Accounts int `json:"accounts"` + AccountsDeleted int `json:"accounts_deleted"` + StorageSlots int `json:"storage_slots"` + StorageSlotsDeleted int `json:"storage_slots_deleted"` + Code int `json:"code"` + CodeBytes int `json:"code_bytes"` +} + +// slowBlockCache represents cache hit/miss statistics for cross-client analysis. +type slowBlockCache struct { + Account slowBlockCacheEntry `json:"account"` + Storage slowBlockCacheEntry `json:"storage"` + Code slowBlockCodeCacheEntry `json:"code"` +} + +// slowBlockCacheEntry represents cache statistics for account/storage caches. +type slowBlockCacheEntry struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` +} + +// slowBlockCodeCacheEntry represents cache statistics for code cache with byte-level granularity. +type slowBlockCodeCacheEntry struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` + HitBytes int64 `json:"hit_bytes"` + MissBytes int64 `json:"miss_bytes"` +} + +// calculateHitRate computes the cache hit rate as a percentage (0-100). +func calculateHitRate(hits, misses int64) float64 { + if total := hits + misses; total > 0 { + return float64(hits) / float64(total) * 100.0 + } + return 0.0 +} + +// durationToMs converts a time.Duration to milliseconds as a float64 +// with sub-millisecond precision for accurate cross-client metrics. +func durationToMs(d time.Duration) float64 { + return float64(d.Nanoseconds()) / 1e6 +} + +// logSlow prints the detailed execution statistics in JSON format if the block +// is regarded as slow. The JSON format is designed for cross-client compatibility +// with other Ethereum execution clients. +func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Duration) { + // Negative threshold means disabled (default when flag not set) + if slowBlockThreshold < 0 { + return + } + // Threshold of 0 logs all blocks; positive threshold filters + if slowBlockThreshold > 0 && s.TotalTime < slowBlockThreshold { + return + } + logEntry := slowBlockLog{ + Level: "warn", + Msg: "Slow block", + Block: slowBlockInfo{ + Number: block.NumberU64(), + Hash: block.Hash(), + GasUsed: block.GasUsed(), + TxCount: len(block.Transactions()), + }, + Timing: slowBlockTime{ + ExecutionMs: durationToMs(s.Execution), + StateReadMs: durationToMs(s.AccountReads + s.StorageReads + s.CodeReads), + StateHashMs: durationToMs(s.AccountHashes + s.AccountUpdates + s.StorageUpdates), + CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.TrieDBCommit + s.SnapshotCommit + s.BlockWrite), + TotalMs: durationToMs(s.TotalTime), + }, + Throughput: slowBlockThru{ + MgasPerSec: s.MgasPerSecond, + }, + StateReads: slowBlockReads{ + Accounts: s.AccountLoaded, + StorageSlots: s.StorageLoaded, + Code: s.CodeLoaded, + CodeBytes: s.CodeLoadBytes, + }, + StateWrites: slowBlockWrites{ + Accounts: s.AccountUpdated, + AccountsDeleted: s.AccountDeleted, + StorageSlots: s.StorageUpdated, + StorageSlotsDeleted: s.StorageDeleted, + Code: s.CodeUpdated, + CodeBytes: s.CodeUpdateBytes, + }, + Cache: slowBlockCache{ + Account: slowBlockCacheEntry{ + Hits: s.StateReadCacheStats.AccountCacheHit, + Misses: s.StateReadCacheStats.AccountCacheMiss, + HitRate: calculateHitRate(s.StateReadCacheStats.AccountCacheHit, s.StateReadCacheStats.AccountCacheMiss), + }, + Storage: slowBlockCacheEntry{ + Hits: s.StateReadCacheStats.StorageCacheHit, + Misses: s.StateReadCacheStats.StorageCacheMiss, + HitRate: calculateHitRate(s.StateReadCacheStats.StorageCacheHit, s.StateReadCacheStats.StorageCacheMiss), + }, + Code: slowBlockCodeCacheEntry{ + Hits: s.StateReadCacheStats.CodeStats.CacheHit, + Misses: s.StateReadCacheStats.CodeStats.CacheMiss, + HitRate: calculateHitRate(s.StateReadCacheStats.CodeStats.CacheHit, s.StateReadCacheStats.CodeStats.CacheMiss), + HitBytes: s.StateReadCacheStats.CodeStats.CacheHitBytes, + MissBytes: s.StateReadCacheStats.CodeStats.CacheMissBytes, + }, + }, + } + jsonBytes, err := json.Marshal(logEntry) + if err != nil { + log.Error("Failed to marshal slow block log", "error", err) + return + } + log.Warn(string(jsonBytes)) +} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b1c71b2f81..c190041bfd 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -18,6 +18,7 @@ package core import ( "bytes" + "context" "errors" "fmt" gomath "math" @@ -160,14 +161,14 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { if err != nil { return err } - res, err := blockchain.processor.Process(block, statedb, vm.Config{}) + res, err := blockchain.processor.Process(context.Background(), block, statedb, vm.Config{}) if err != nil { - blockchain.reportBlock(block, res, err) + blockchain.reportBadBlock(block, res, err) return err } err = blockchain.validator.ValidateState(block, statedb, res, false) if err != nil { - blockchain.reportBlock(block, res, err) + blockchain.reportBadBlock(block, res, err) return err } @@ -3456,7 +3457,7 @@ func testSetCanonical(t *testing.T, scheme string) { gen.AddTx(tx) }) for _, block := range side { - _, err := chain.InsertBlockWithoutSetHead(block, false) + _, err := chain.InsertBlockWithoutSetHead(context.Background(), block, false) if err != nil { t.Fatalf("Failed to insert into chain: %v", err) } @@ -4515,3 +4516,45 @@ func TestGetCanonicalReceipt(t *testing.T) { } } } + +// TestSetHeadBeyondRootFinalizedBug tests the issue where the finalized block +// is not cleared when rewinding past it using setHeadBeyondRoot. +func TestSetHeadBeyondRootFinalizedBug(t *testing.T) { + // Create a clean blockchain with 100 blocks using PathScheme (PBSS) + _, _, blockchain, err := newCanonical(ethash.NewFaker(), 100, true, rawdb.PathScheme) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() + + // Set the "Finalized" marker to the current Head (Block 100) + headBlock := blockchain.CurrentBlock() + if headBlock.Number.Uint64() != 100 { + t.Fatalf("Setup failed: expected head 100, got %d", headBlock.Number.Uint64()) + } + blockchain.SetFinalized(headBlock) + + // Verify setup + if blockchain.CurrentFinalBlock().Number.Uint64() != 100 { + t.Fatalf("Setup failed: Finalized block should be 100") + } + targetBlock := blockchain.GetBlockByNumber(50) + + // Call setHeadBeyondRoot with: + // head = 100 + // repair = true + if _, _, err := blockchain.setHeadBeyondRoot(100, 0, targetBlock.Root(), true, 0); err != nil { + t.Fatalf("Failed to rewind: %v", err) + } + + currentFinal := blockchain.CurrentFinalBlock() + currentHead := blockchain.CurrentBlock().Number.Uint64() + + // The previous finalized block (100) is now invalid because we rewound to 50. + // The function should have cleared the finalized marker (set to nil). + if currentFinal != nil && currentFinal.Number.Uint64() > currentHead { + t.Errorf("Chain Head: %d , Finalized Block: %d , Finalized block was >= head block.", + currentHead, + currentFinal.Number.Uint64()) + } +} diff --git a/core/chain_makers.go b/core/chain_makers.go index ea91ee7970..4d6e0bf463 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-verkle" "github.com/holiman/uint256" ) @@ -430,7 +429,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := triedb.NewDatabase(db, triedb.HashDefaults) + var triedbConfig *triedb.Config = triedb.HashDefaults + if config.IsVerkle(config.ChainID, 0) { + triedbConfig = triedb.VerkleDefaults + } + triedb := triedb.NewDatabase(db, triedbConfig) defer triedb.Close() for i := 0; i < n; i++ { @@ -475,9 +478,13 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - triedb := triedb.NewDatabase(db, triedb.HashDefaults) + var triedbConfig *triedb.Config = triedb.HashDefaults + if genesis.Config != nil && genesis.Config.IsVerkle(genesis.Config.ChainID, 0) { + triedbConfig = triedb.VerkleDefaults + } + triedb := triedb.NewDatabase(db, triedbConfig) defer triedb.Close() - _, err := genesis.Commit(db, triedb) + _, err := genesis.Commit(db, triedb, nil) if err != nil { panic(err) } @@ -485,117 +492,6 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, return db, blocks, receipts } -func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - if config == nil { - config = params.TestChainConfig - } - proofs := make([]*verkle.VerkleProof, 0, n) - keyvals := make([]verkle.StateDiff, 0, n) - cm := newChainMaker(parent, config, engine) - - genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { - b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} - b.header = cm.makeHeader(parent, statedb, b.engine) - - // TODO uncomment when proof generation is merged - // Save pre state for proof generation - // preState := statedb.Copy() - - // EIP-2935 / 7709 - blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - blockContext.Random = &common.Hash{} // enable post-merge instruction set - evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm) - - // Execute any user modifications to the block. - if gen != nil { - gen(i, b) - } - - requests := b.collectRequests(false) - if requests != nil { - reqHash := types.CalcRequestsHash(requests) - b.header.RequestsHash = &reqHash - } - - body := &types.Body{ - Transactions: b.txs, - Uncles: b.uncles, - Withdrawals: b.withdrawals, - } - block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, body, b.receipts) - if err != nil { - panic(err) - } - - // Write state changes to DB. - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), config.IsCancun(b.header.Number, b.header.Time, types.DeserializeHeaderExtraInformation(b.header).ArbOSFormatVersion)) - if err != nil { - panic(fmt.Sprintf("state write error: %v", err)) - } - if err = triedb.Commit(root, false); err != nil { - panic(fmt.Sprintf("trie write error: %v", err)) - } - - proofs = append(proofs, block.ExecutionWitness().VerkleProof) - keyvals = append(keyvals, block.ExecutionWitness().StateDiff) - - return block, b.receipts - } - - sdb := state.NewDatabase(trdb, nil) - - for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), sdb) - if err != nil { - panic(err) - } - block, receipts := genblock(i, parent, trdb, statedb) - - // Post-process the receipts. - // Here we assign the final block hash and other info into the receipt. - // In order for DeriveFields to work, the transaction and receipt lists need to be - // of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be - // extra ones, so we just trim the lists here. - receiptsCount := len(receipts) - txs := block.Transactions() - if len(receipts) > len(txs) { - receipts = receipts[:len(txs)] - } else if len(receipts) < len(txs) { - txs = txs[:len(receipts)] - } - var blobGasPrice *big.Int - if block.ExcessBlobGas() != nil { - blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header()) - } - if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs, true); err != nil { - panic(err) - } - - // Re-expand to ensure all receipts are returned. - receipts = receipts[:receiptsCount] - - // Advance the chain. - cm.add(block, receipts) - parent = block - } - return cm.chain, cm.receipts, proofs, keyvals -} - -func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - defer triedb.Close() - genesisBlock, err := genesis.Commit(db, triedb) - if err != nil { - panic(err) - } - blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen) - return genesisBlock.Hash(), db, blocks, receipts, proofs, keyvals -} - func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { time := parent.Time() + 10 // block time is fixed at 10 seconds parentHeader := parent.Header() diff --git a/core/events.go b/core/events.go index ef0de32426..ed853f1790 100644 --- a/core/events.go +++ b/core/events.go @@ -17,6 +17,9 @@ package core import ( + "time" + + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -35,3 +38,10 @@ type ChainEvent struct { type ChainHeadEvent struct { Header *types.Header } + +// NewPayloadEvent is posted when engine_newPayloadVX processes a block. +type NewPayloadEvent struct { + Hash common.Hash + Number uint64 + ProcessingTime time.Duration +} diff --git a/core/evm.go b/core/evm.go index 0a827aa56b..04c7b72045 100644 --- a/core/evm.go +++ b/core/evm.go @@ -86,12 +86,9 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common func NewEVMTxContext(msg *Message) vm.TxContext { ctx := vm.TxContext{ Origin: msg.From, - GasPrice: new(big.Int).Set(msg.GasPrice), + GasPrice: uint256.MustFromBig(msg.GasPrice), BlobHashes: msg.BlobHashes, } - if msg.BlobGasFeeCap != nil { - ctx.BlobFeeCap = new(big.Int).Set(msg.BlobGasFeeCap) - } return ctx } diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index 2ea065ddb7..795967405d 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -288,5 +288,32 @@ {"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424}, {"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273}, {"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627}, -{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073} +{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}, +{"blockNumber": 23078983, "blockId": "0xd64da4fe45a0a349101b8ce1a6336fb099d8b00cc274d0eb59356e134190b8f2", "firstIndex": 19528679292}, +{"blockNumber": 23109224, "blockId": "0xbb6c29f91820fcf6caef881bdfe61eb690f9796f8f139c56eaf27aa601fe5ed2", "firstIndex": 19595787850}, +{"blockNumber": 23136690, "blockId": "0xc1915739edff731d469ec1500ad05102c0d68cc1ef062411d2e9741b8ebdc571", "firstIndex": 19662896969}, +{"blockNumber": 23164324, "blockId": "0x97d0078f2a22a8fbde4660d5f11846d00a669c606e42291df55e863190914a9f", "firstIndex": 19730004997}, +{"blockNumber": 23192975, "blockId": "0x8463749ec09f55fccdd962498f93d6468d744faedc35da8097642cd4609e08f2", "firstIndex": 19797111604}, +{"blockNumber": 23219805, "blockId": "0x76ca51d01b5724f1b0f6c3acb50dfcdb00e1648a475dd7599174b301c25e5517", "firstIndex": 19864222312}, +{"blockNumber": 23236331, "blockId": "0x388f9e36b3ec2120d2b4adfdcdb0c0b8de139eef107e327670dd77fc581c2c5f", "firstIndex": 19931331831}, +{"blockNumber": 23260849, "blockId": "0xe4a467164dbc8f9beebf0c8478a9e7ee16dfce65e2da423b1048601831222ba7", "firstIndex": 19998441286}, +{"blockNumber": 23282795, "blockId": "0x74ad210aa1bfdd4bcf61298ebff622da758c36c38d62d001f2440d09e73ef6c7", "firstIndex": 20065548083}, +{"blockNumber": 23300759, "blockId": "0xa405f5ea21a5207d3cde718a1e3fb7f0ce3dd87ac6040a0db52da0e9488e63f6", "firstIndex": 20132623334}, +{"blockNumber": 23319772, "blockId": "0xafae645dd057af450eddf69c7604bde0136524abc5b7d6697426427ef2d30724", "firstIndex": 20199767615}, +{"blockNumber": 23342113, "blockId": "0x8482c4be13294cfd862d500051c8b4efb95b50f4a79717da6aeabb2a6ff3e199", "firstIndex": 20266874986}, +{"blockNumber": 23366974, "blockId": "0xf6047cafea5da7aaf20691df700d759fe84f5aa2176d9dd42c6ae138899a29ea", "firstIndex": 20333983351}, +{"blockNumber": 23397579, "blockId": "0xe94815fe0278659a26e15b023c0c3877abf6f1265710c5dfddf161ee8af01b40", "firstIndex": 20401094335}, +{"blockNumber": 23425940, "blockId": "0xa98d6d48b93d9ef848c340b47cf9d191f36e76375dd747d8adcb39d72251822d", "firstIndex": 20468200506}, +{"blockNumber": 23452402, "blockId": "0x5f26ff308c0883a04f4a92411bcd05e50538d95a96532452f43b71fde4af908d", "firstIndex": 20535311178}, +{"blockNumber": 23478884, "blockId": "0x23a81186c21218f7034ee81dcd72db71edcbc0e97da26703b8026b6d4a81f693", "firstIndex": 20602384106}, +{"blockNumber": 23507136, "blockId": "0xb1d4a2bd836c5d78b678f0bfffd651bdfeae8b7965639b640f8745193b142323", "firstIndex": 20669529935}, +{"blockNumber": 23535463, "blockId": "0x1886a719a6376352753242ec5093c434938b9a90356facdbdaafd2670da97d82", "firstIndex": 20736637374}, +{"blockNumber": 23564278, "blockId": "0xcdc5a1349f45430771fa2080555f20b80b2d84f575728dbb8bd3b633fbb0a00b", "firstIndex": 20803747399}, +{"blockNumber": 23594711, "blockId": "0xbbf41e407367feeb865b305e38aee8bef84ba62b66efbd6fcd3b16a5d50cc055", "firstIndex": 20870854523}, +{"blockNumber": 23627417, "blockId": "0x1b3c6bd39c5aa7c73101e7e946964b3d7afecf3f253efda8de5610dd99cbee82", "firstIndex": 20937963176}, +{"blockNumber": 23660347, "blockId": "0xadc8ac88c281f50c3aaf1d08149100e8f91a16bbe09b28ac86665bcea681d41b", "firstIndex": 21005072865}, +{"blockNumber": 23692228, "blockId": "0xeb4040468161d9d5b6863eb62e4f21f72a1944f910ecfa95ee6a9dbc39e92ef0", "firstIndex": 21072181753}, +{"blockNumber": 23722331, "blockId": "0xdfc9be1b43488148868da0c8ac56e99e0ffde18bcc418755c4765e745fe74024", "firstIndex": 21139291342}, +{"blockNumber": 23752866, "blockId": "0xb9cf0dfee429a1450f5fb7a237729cf7d0c49e7e35c00dc3709f197c091f8b39", "firstIndex": 21206399901}, +{"blockNumber": 23784485, "blockId": "0x92f5f119078b4ef7ad99273e2ae6f874cfb663e80d741a821e0bb7c25c0369c7", "firstIndex": 21273505141} ] diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index fede54df57..f6b1ef26d0 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -434,7 +434,7 @@ func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, ha lastLogPrinted = start ) switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { - if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + if deleted && (!logPrinted || time.Since(lastLogPrinted) > time.Second*10) { log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) logPrinted, lastLogPrinted = true, time.Now() } diff --git a/core/genesis.go b/core/genesis.go index a31d2eae56..fbdfa2061e 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -183,7 +183,7 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { // flushAlloc is very similar with hash, but the main difference is all the // generated states will be persisted into the given database. -func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, error) { +func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database, tracer *tracing.Hooks) (common.Hash, error) { emptyRoot := types.EmptyRootHash if triedb.IsVerkle() { emptyRoot = types.EmptyVerkleHash @@ -204,12 +204,28 @@ func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, e statedb.SetState(addr, key, value) } } - root, err := statedb.Commit(0, false, false) - if err != nil { - return common.Hash{}, err + + var root common.Hash + if tracer != nil && tracer.OnStateUpdate != nil { + r, update, err := statedb.CommitWithUpdate(0, false, false) + if err != nil { + return common.Hash{}, err + } + trUpdate, err := update.ToTracingUpdate() + if err != nil { + return common.Hash{}, err + } + tracer.OnStateUpdate(trUpdate) + root = r + } else { + root, err = statedb.Commit(0, false, false) + if err != nil { + return common.Hash{}, err + } } + // Commit newly generated states into disk if it's not empty. - if root != types.EmptyRootHash { + if root != emptyRoot { if err := triedb.Commit(root, true); err != nil { return common.Hash{}, err } @@ -315,10 +331,10 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { // specify a fork block below the local head block). In case of a conflict, the // error is a *params.ConfigCompatError and the new, unwritten config is returned. func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { - return SetupGenesisBlockWithOverride(db, triedb, genesis, nil) + return SetupGenesisBlockWithOverride(db, triedb, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides, tracer *tracing.Hooks) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { // Copy the genesis, so we can operate on a copy. genesis = genesis.copy() // Sanitize the supplied genesis, ensuring it has the associated chain @@ -339,7 +355,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g return nil, common.Hash{}, nil, err } - block, err := genesis.Commit(db, triedb) + block, err := genesis.Commit(db, triedb, tracer) if err != nil { return nil, common.Hash{}, nil, err } @@ -367,7 +383,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if hash := genesis.ToBlock().Hash(); hash != ghash { return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash} } - block, err := genesis.Commit(db, triedb) + block, err := genesis.Commit(db, triedb, tracer) if err != nil { return nil, common.Hash{}, nil, err } @@ -557,7 +573,7 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { +func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database, tracer *tracing.Hooks) (*types.Block, error) { if g.Number != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -572,7 +588,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo return nil, errors.New("can't start clique chain without signers") } // flush the data to disk and compute the state root - root, err := flushAlloc(&g.Alloc, triedb) + root, err := flushAlloc(&g.Alloc, triedb, tracer) if err != nil { return nil, err } @@ -602,7 +618,7 @@ func WriteHeadBlock(batch ethdb.Batch, block *types.Block) { // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block { - block, err := g.Commit(db, triedb) + block, err := g.Commit(db, triedb, nil) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index abc623d00f..400a6a0c82 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -88,7 +88,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, nil) }, wantHash: customghash, @@ -98,7 +98,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == sepolia", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, DefaultSepoliaGenesisBlock()) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.SepoliaGenesisHash}, @@ -107,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == hoodi", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) + customg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, DefaultHoodiGenesisBlock()) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.HoodiGenesisHash}, @@ -116,7 +116,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb) + oldcustomg.Commit(db, tdb, nil) return SetupGenesisBlock(db, tdb, &customg) }, wantHash: customghash, @@ -128,7 +128,7 @@ func testSetupGenesis(t *testing.T, scheme string) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. tdb := triedb.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb) + oldcustomg.Commit(db, tdb, nil) bc, _ := NewBlockChain(db, nil, &oldcustomg, ethash.NewFullFaker(), DefaultConfig().WithStateScheme(scheme)) defer bc.Stop() @@ -308,7 +308,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("018d20eebb130b5e2b796465fe36aafab650650729a92435aec071bf2386f080") + expected := common.FromHex("b94812c1674dcf4f2bc98f4503d15f4cc674265135bcf3be6e4417b60881042a") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/headerchain_test.go b/core/headerchain_test.go index b51fb8f226..dba04e2cf2 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -69,7 +69,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, triedb.NewDatabase(db, nil)) + gspec.Commit(db, triedb.NewDatabase(db, nil), nil) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index 67ca0f9671..a52d9139c9 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -97,7 +97,7 @@ func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle boo // Initialize the first transition state, with the "ended" // field set to true if the database was created // as a verkle database. - log.Debug("no transition state found, starting fresh", "is verkle", db) + log.Debug("no transition state found, starting fresh", "verkle", isVerkle) // Start with a fresh state ts = &TransitionState{Ended: isVerkle} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 5235d01bc3..39a0e32f43 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -83,65 +83,6 @@ type NumberHash struct { Hash common.Hash } -// ReadAllHashesInRange retrieves all the hashes assigned to blocks at certain -// heights, both canonical and reorged forks included. -// This method considers both limits to be _inclusive_. -func ReadAllHashesInRange(db ethdb.Iteratee, first, last uint64) []*NumberHash { - var ( - start = encodeBlockNumber(first) - keyLength = len(headerPrefix) + 8 + 32 - hashes = make([]*NumberHash, 0, 1+last-first) - it = db.NewIterator(headerPrefix, start) - ) - defer it.Release() - for it.Next() { - key := it.Key() - if len(key) != keyLength { - continue - } - num := binary.BigEndian.Uint64(key[len(headerPrefix) : len(headerPrefix)+8]) - if num > last { - break - } - hash := common.BytesToHash(key[len(key)-32:]) - hashes = append(hashes, &NumberHash{num, hash}) - } - return hashes -} - -// ReadAllCanonicalHashes retrieves all canonical number and hash mappings at the -// certain chain range. If the accumulated entries reaches the given threshold, -// abort the iteration and return the semi-finish result. -func ReadAllCanonicalHashes(db ethdb.Iteratee, from uint64, to uint64, limit int) ([]uint64, []common.Hash) { - // Short circuit if the limit is 0. - if limit == 0 { - return nil, nil - } - var ( - numbers []uint64 - hashes []common.Hash - ) - // Construct the key prefix of start point. - start, end := headerHashKey(from), headerHashKey(to) - it := db.NewIterator(nil, start) - defer it.Release() - - for it.Next() { - if bytes.Compare(it.Key(), end) >= 0 { - break - } - if key := it.Key(); len(key) == len(headerPrefix)+8+1 && bytes.Equal(key[len(key)-1:], headerHashSuffix) { - numbers = append(numbers, binary.BigEndian.Uint64(key[len(headerPrefix):len(headerPrefix)+8])) - hashes = append(hashes, common.BytesToHash(it.Value())) - // If the accumulated entries reaches the limit threshold, return. - if len(numbers) >= limit { - break - } - } - } - return numbers, hashes -} - // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) (uint64, bool) { data, _ := db.Get(headerNumberKey(hash)) @@ -483,7 +424,13 @@ func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but bodies can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false @@ -525,7 +472,13 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // to a block. func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but receipts can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { return false @@ -921,40 +874,6 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { } } -// DeleteBadBlocks deletes all the bad blocks from the database -func DeleteBadBlocks(db ethdb.KeyValueWriter) { - if err := db.Delete(badBlockKey); err != nil { - log.Crit("Failed to delete bad blocks", "err", err) - } -} - -// FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header { - for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { - a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - } - for an := a.Number.Uint64(); an < b.Number.Uint64(); { - b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - for a.Hash() != b.Hash() { - a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - return a -} - // ReadHeadHeader returns the current canonical head header. func ReadHeadHeader(db ethdb.Reader) *types.Header { headHeaderHash := ReadHeadHeaderHash(db) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index cc2431ca0d..da6e618141 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -23,7 +23,6 @@ import ( "math/big" "math/rand" "os" - "reflect" "testing" "github.com/ethereum/go-ethereum/arbcrypto" @@ -53,10 +52,7 @@ func TestHeaderStorage(t *testing.T) { if entry := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { - hasher := arbcrypto.NewLegacyKeccak256() - hasher.Write(entry) - - if hash := common.BytesToHash(hasher.Sum(nil)); hash != header.Hash() { + if hash := crypto.Keccak256Hash(entry); hash != header.Hash() { t.Fatalf("Retrieved RLP header mismatch: have %v, want %v", entry, header) } } @@ -91,10 +87,7 @@ func TestBodyStorage(t *testing.T) { if entry := ReadBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { - hasher := arbcrypto.NewLegacyKeccak256() - hasher.Write(entry) - - if calc := common.BytesToHash(hasher.Sum(nil)); calc != hash { + if calc := crypto.Keccak256Hash(entry); calc != hash { t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) } } @@ -249,13 +242,6 @@ func TestBadBlockStorage(t *testing.T) { t.Fatalf("The bad blocks are not sorted #[%d](%d) < #[%d](%d)", i, badBlocks[i].NumberU64(), i+1, badBlocks[i+1].NumberU64()) } } - - // Delete all bad blocks - DeleteBadBlocks(db) - badBlocks = ReadAllBadBlocks(db) - if len(badBlocks) != 0 { - t.Fatalf("Failed to delete bad blocks") - } } // Tests that canonical numbers can be mapped to hashes and retrieved. @@ -516,37 +502,6 @@ func TestWriteAncientHeaderChain(t *testing.T) { } } -func TestCanonicalHashIteration(t *testing.T) { - var cases = []struct { - from, to uint64 - limit int - expect []uint64 - }{ - {1, 8, 0, nil}, - {1, 8, 1, []uint64{1}}, - {1, 8, 10, []uint64{1, 2, 3, 4, 5, 6, 7}}, - {1, 9, 10, []uint64{1, 2, 3, 4, 5, 6, 7, 8}}, - {2, 9, 10, []uint64{2, 3, 4, 5, 6, 7, 8}}, - {9, 10, 10, nil}, - } - // Test empty db iteration - db := NewMemoryDatabase() - numbers, _ := ReadAllCanonicalHashes(db, 0, 10, 10) - if len(numbers) != 0 { - t.Fatalf("No entry should be returned to iterate an empty db") - } - // Fill database with testing data. - for i := uint64(1); i <= 8; i++ { - WriteCanonicalHash(db, common.Hash{}, i) - } - for i, c := range cases { - numbers, _ := ReadAllCanonicalHashes(db, c.from, c.to, c.limit) - if !reflect.DeepEqual(numbers, c.expect) { - t.Fatalf("Case %d failed, want %v, got %v", i, c.expect, numbers) - } - } -} - func TestHashesInRange(t *testing.T) { mkHeader := func(number, seq int) *types.Header { h := types.Header{ @@ -565,18 +520,6 @@ func TestHashesInRange(t *testing.T) { total++ } } - if have, want := len(ReadAllHashesInRange(db, 10, 10)), 10; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 10, 9)), 0; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 0, 100)), total; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } - if have, want := len(ReadAllHashesInRange(db, 9, 10)), 9+10; have != want { - t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) - } if have, want := len(ReadAllHashes(db, 10)), 10; have != want { t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 535178628b..827708ec9c 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -147,9 +147,6 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex := uint64(0) for iter.Next() { - if iter.Err() != nil { - return nil, 0, err - } // The preimage for the hash calculation of legacy transactions // is just their RLP encoding. For typed (EIP-2718) transactions, // which are encoded as byte arrays, the preimage is the content of @@ -177,6 +174,9 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex++ } + if iter.Err() != nil { + return nil, 0, iter.Err() + } return nil, 0, errors.New("transaction not found") } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index b940d91040..8c6b18df08 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -34,14 +34,10 @@ type freezerInfo struct { name string // The identifier of freezer head uint64 // The number of last stored item in the freezer tail uint64 // The number of first stored item in the freezer + count uint64 // The number of stored items in the freezer sizes []tableSize // The storage size per table } -// count returns the number of stored items in the freezer. -func (info *freezerInfo) count() uint64 { - return info.head - info.tail + 1 -} - // size returns the storage size of the entire freezer. func (info *freezerInfo) size() common.StorageSize { var total common.StorageSize @@ -65,7 +61,11 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci if err != nil { return freezerInfo{}, err } - info.head = ancients - 1 + if ancients > 0 { + info.head = ancients - 1 + } else { + info.head = 0 + } // Retrieve the number of first stored item tail, err := reader.Tail() @@ -73,6 +73,12 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci return freezerInfo{}, err } info.tail = tail + + if ancients == 0 { + info.count = 0 + } else { + info.count = info.head - info.tail + 1 + } return info, nil } @@ -143,6 +149,8 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs case MerkleStateFreezerName, VerkleStateFreezerName: path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs + case MerkleTrienodeFreezerName, VerkleTrienodeFreezerName: + path, tables = filepath.Join(ancient, freezerName), trienodeFreezerTableConfigs default: return fmt.Errorf("unknown freezer, supported ones: %v", freezers) } @@ -158,6 +166,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s if err != nil { return err } + defer table.Close() table.dumpIndexStdout(start, end) return nil } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 3eb5500f2f..ad7ba5fc3d 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -94,6 +94,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { type blockTxHashes struct { number uint64 hashes []common.Hash + err error } // iterateTransactions iterates over all transactions in the (canon) block @@ -157,17 +158,22 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool }() for data := range rlpCh { var body types.Body + var result *blockTxHashes if err := rlp.DecodeBytes(data.rlp, &body); err != nil { log.Warn("Failed to decode block body", "block", data.number, "error", err) - return - } - var hashes []common.Hash - for _, tx := range body.Transactions { - hashes = append(hashes, tx.Hash()) - } - result := &blockTxHashes{ - hashes: hashes, - number: data.number, + result = &blockTxHashes{ + number: data.number, + err: err, + } + } else { + hashes := make([]common.Hash, len(body.Transactions)) + for i, tx := range body.Transactions { + hashes[i] = tx.Hash() + } + result = &blockTxHashes{ + hashes: hashes, + number: data.number, + } } // Feed the block to the aggregator, or abort on interrupt select { @@ -227,6 +233,10 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan // Next block available, pop it off and index it delivery := queue.PopItem() lastNum = delivery.number + if delivery.err != nil { + log.Warn("Skipping tx indexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } WriteTxLookupEntries(batch, delivery.number, delivery.hashes) blocks++ txs += len(delivery.hashes) @@ -322,6 +332,10 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } delivery := queue.PopItem() nextNum = delivery.number + 1 + if delivery.err != nil { + log.Warn("Skipping tx unindexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } DeleteTxLookupEntries(batch, delivery.hashes) txs += len(delivery.hashes) blocks++ @@ -360,9 +374,9 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } select { case <-interrupt: - logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) } txIndexerUnindexedBlocksHistogram.Update(int64(blocks)) txIndexerUnindexedTransactionsHistogram.Update(int64(txs)) diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index c1adfd3e80..6774842e32 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -218,6 +218,36 @@ func TestIndexTransactions(t *testing.T) { verify(0, 8, false, 8) } +func TestUnindexTransactionsMissingBody(t *testing.T) { + // Construct test chain db + chainDB := NewMemoryDatabase() + blocks, _ := initDatabaseWithTransactions(chainDB) + + // Index the entire chain. + lastBlock := blocks[len(blocks)-1].NumberU64() + IndexTransactions(chainDB, 0, lastBlock+1, nil, false, 0) + + // Prove that block 2 body exists in the database. + if raw := ReadCanonicalBodyRLP(chainDB, 2, nil); len(raw) == 0 { + t.Fatalf("Block 2 body does not exist in the database.") + } + + // Delete body for block 2. This simulates a corrupted database. + key := blockBodyKey(2, blocks[2].Hash()) + if err := chainDB.Delete(key); err != nil { + t.Fatalf("Failed to delete block body %v", err) + } + + // Unindex blocks [0, 3) + UnindexTransactions(chainDB, 0, 3, nil, false, 0) + + // Verify that tx index tail is updated to 3. + tail := ReadTxIndexTail(chainDB) + if tail == nil || *tail != 3 { + t.Fatalf("The tx index tail is wrong: got %v want %d", *tail, 3) + } +} + func TestPruneTransactionIndex(t *testing.T) { chainDB := NewMemoryDatabase() blocks, _ := initDatabaseWithTransactions(chainDB) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 78a30d01b0..cc1542d69d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -368,6 +368,7 @@ func Open(db ethdb.KeyValueStore, opts OpenOptions) (ethdb.Database, error) { }() } return &freezerdb{ + readOnly: opts.ReadOnly, ancientRoot: opts.Ancient, KeyValueStore: db, chainFreezer: frdb, @@ -466,7 +467,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { filterMapBlockLV stat // Path-mode archive data - stateIndex stat + stateIndex stat + trienodeIndex stat // Verkle statistics verkleTries stat @@ -561,8 +563,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bloomBits.add(size) // Path-based historic state indexes - case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: + case bytes.HasPrefix(key, StateHistoryAccountMetadataPrefix) && len(key) == len(StateHistoryAccountMetadataPrefix)+common.HashLength: stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageMetadataPrefix) && len(key) == len(StateHistoryStorageMetadataPrefix)+2*common.HashLength: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryAccountBlockPrefix) && len(key) == len(StateHistoryAccountBlockPrefix)+common.HashLength+4: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageBlockPrefix) && len(key) == len(StateHistoryStorageBlockPrefix)+2*common.HashLength+4: + stateIndex.add(size) + + case bytes.HasPrefix(key, TrienodeHistoryMetadataPrefix) && len(key) >= len(TrienodeHistoryMetadataPrefix)+common.HashLength: + trienodeIndex.add(size) + case bytes.HasPrefix(key, TrienodeHistoryBlockPrefix) && len(key) >= len(TrienodeHistoryBlockPrefix)+common.HashLength+4: + trienodeIndex.add(size) // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): @@ -659,12 +672,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()}, {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()}, {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()}, - {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()}, {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()}, {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()}, {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()}, {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()}, {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()}, + {"Key-Value store", "Historical state index", stateIndex.sizeString(), stateIndex.countString()}, + {"Key-Value store", "Historical trie index", trienodeIndex.sizeString(), trienodeIndex.countString()}, {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()}, {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()}, @@ -681,13 +695,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), strings.Title(table.name), table.size.String(), - fmt.Sprintf("%d", ancient.count()), + fmt.Sprintf("%d", ancient.count), }) } total.Add(uint64(ancient.size())) } - table := newTableWriter(os.Stdout) + table := NewTableWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) @@ -709,7 +723,7 @@ var knownMetadataKeys = [][]byte{ snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, - filterMapsRangeKey, headStateHistoryIndexKey, VerkleTransitionStatePrefix, + filterMapsRangeKey, headStateHistoryIndexKey, headTrienodeHistoryIndexKey, VerkleTransitionStatePrefix, } // printChainMetadata prints out chain metadata to stderr. diff --git a/core/rawdb/database_tablewriter_tinygo.go b/core/rawdb/database_tablewriter.go similarity index 93% rename from core/rawdb/database_tablewriter_tinygo.go rename to core/rawdb/database_tablewriter.go index 2f8e456fd5..e1cda5c93f 100644 --- a/core/rawdb/database_tablewriter_tinygo.go +++ b/core/rawdb/database_tablewriter.go @@ -14,10 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// TODO: naive stub implementation for tablewriter - -//go:build tinygo -// +build tinygo +// Naive stub implementation for tablewriter package rawdb @@ -40,7 +37,7 @@ type Table struct { rows [][]string } -func newTableWriter(w io.Writer) *Table { +func NewTableWriter(w io.Writer) *Table { return &Table{out: w} } @@ -89,6 +86,7 @@ func (t *Table) render() error { rowSeparator := t.buildRowSeparator(widths) if len(t.headers) > 0 { + fmt.Fprintln(t.out, rowSeparator) t.printRow(t.headers, widths) fmt.Fprintln(t.out, rowSeparator) } @@ -100,6 +98,7 @@ func (t *Table) render() error { if len(t.footer) > 0 { fmt.Fprintln(t.out, rowSeparator) t.printRow(t.footer, widths) + fmt.Fprintln(t.out, rowSeparator) } return nil @@ -172,21 +171,22 @@ func (t *Table) calculateColumnWidths() []int { // // It generates a string with dashes (-) for each column width, joined by plus signs (+). // -// Example output: "----------+--------+-----------" +// Example output: "+----------+--------+-----------+" func (t *Table) buildRowSeparator(widths []int) string { parts := make([]string, len(widths)) for i, w := range widths { parts[i] = strings.Repeat("-", w) } - return strings.Join(parts, "+") + return "+" + strings.Join(parts, "+") + "+" } // printRow outputs a single row to the table writer. // // Each cell is padded with spaces and separated by pipe characters (|). // -// Example output: " Database | Size | Items " +// Example output: "| Database | Size | Items |" func (t *Table) printRow(row []string, widths []int) { + fmt.Fprintf(t.out, "|") for i, cell := range row { if i > 0 { fmt.Fprint(t.out, "|") @@ -204,5 +204,6 @@ func (t *Table) printRow(row []string, widths []int) { fmt.Fprintf(t.out, "%s%s%s", leftPadding, cell, rightPadding) } + fmt.Fprintf(t.out, "|") fmt.Fprintln(t.out) } diff --git a/core/rawdb/database_tablewriter_tinygo_test.go b/core/rawdb/database_tablewriter_test.go similarity index 94% rename from core/rawdb/database_tablewriter_tinygo_test.go rename to core/rawdb/database_tablewriter_test.go index 3bcf93832b..e9de5d8ce8 100644 --- a/core/rawdb/database_tablewriter_tinygo_test.go +++ b/core/rawdb/database_tablewriter_test.go @@ -14,9 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build tinygo -// +build tinygo - package rawdb import ( @@ -27,7 +24,7 @@ import ( func TestTableWriterTinyGo(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"Database", "Size", "Items", "Status"} rows := [][]string{ @@ -51,7 +48,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test missing headers t.Run("MissingHeaders", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) rows := [][]string{{"x", "y", "z"}} @@ -66,7 +63,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("NotEnoughRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -85,7 +82,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("TooManyRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -105,7 +102,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test mismatched footer columns t.Run("MismatchedFooterColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} rows := [][]string{{"x", "y", "z"}} diff --git a/core/rawdb/eradb/eradb.go b/core/rawdb/eradb/eradb.go index 29e658798e..d715c824ed 100644 --- a/core/rawdb/eradb/eradb.go +++ b/core/rawdb/eradb/eradb.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) @@ -51,7 +52,7 @@ type Store struct { type fileCacheEntry struct { refcount int // reference count. This is protected by Store.mu! opened chan struct{} // signals opening of file has completed - file *era.Era // the file + file *onedb.Era // the file err error // error from opening the file } @@ -102,7 +103,7 @@ func (db *Store) Close() { // GetRawBody returns the raw body for a given block number. func (db *Store) GetRawBody(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -117,7 +118,7 @@ func (db *Store) GetRawBody(number uint64) ([]byte, error) { // GetRawReceipts returns the raw receipts for a given block number. func (db *Store) GetRawReceipts(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -249,7 +250,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC } // fileOpened is called after an era file has been successfully opened. -func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *era.Era) { +func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) { db.mu.Lock() defer db.mu.Unlock() @@ -282,7 +283,7 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error entry.err = err } -func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { +func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) { // File name scheme is --. glob := fmt.Sprintf("*-%05d-*.era1", epoch) matches, err := filepath.Glob(filepath.Join(db.datadir, glob)) @@ -297,16 +298,17 @@ func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { } filename := matches[0] - e, err := era.Open(filename) + e, err := onedb.Open(filename) if err != nil { return nil, err } // Sanity-check start block. - if e.Start()%uint64(era.MaxEra1Size) != 0 { - return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxEra1Size) + if e.Start()%uint64(era.MaxSize) != 0 { + e.Close() + return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize) } log.Debug("Opened era1 file", "epoch", epoch) - return e, nil + return e.(*onedb.Era), nil } // doneWithFile signals that the caller has finished using a file. diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 7e46e49f43..080c0720a1 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -51,12 +51,18 @@ func newFreezerBatch(f *Freezer) *freezerBatch { // Append adds an RLP-encoded item of the given kind. func (batch *freezerBatch) Append(kind string, num uint64, item interface{}) error { - return batch.tables[kind].Append(num, item) + if table := batch.tables[kind]; table != nil { + return table.Append(num, item) + } + return errUnknownTable } // AppendRaw adds an item of the given kind. func (batch *freezerBatch) AppendRaw(kind string, num uint64, item []byte) error { - return batch.tables[kind].AppendRaw(num, item) + if table := batch.tables[kind]; table != nil { + return table.AppendRaw(num, item) + } + return errUnknownTable } // reset initializes the batch. diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 8cb4cc2006..a0d308f896 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -91,6 +91,13 @@ func (t *memoryTable) truncateHead(items uint64) error { if items < t.offset { return errors.New("truncation below tail") } + for i := int(items - t.offset); i < len(t.data); i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } + } t.data = t.data[:items-t.offset] t.items = items return nil @@ -108,6 +115,13 @@ func (t *memoryTable) truncateTail(items uint64) error { if t.items < items { return errors.New("truncation above head") } + for i := uint64(0); i < items-t.offset; i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } + } t.data = t.data[items-t.offset:] t.offset = items return nil diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index f531e668c3..5494a648c8 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -221,13 +221,12 @@ func cleanup(path string) error { if err != nil { return err } + defer dir.Close() + names, err := dir.Readdirnames(0) if err != nil { return err } - if cerr := dir.Close(); cerr != nil { - return cerr - } for _, name := range names { if name == filepath.Base(path)+tmpSuffix { log.Info("Removed leftover freezer directory", "name", name) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 01a754c5c8..aedb2d8eed 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -1196,8 +1196,7 @@ func (t *freezerTable) sizeNolock() (uint64, error) { } // advanceHead should be called when the current head file would outgrow the file limits, -// and a new file must be opened. The caller of this method must hold the write-lock -// before calling this method. +// and a new file must be opened. This method acquires the write-lock internally. func (t *freezerTable) advanceHead() error { t.lock.Lock() defer t.lock.Unlock() @@ -1218,7 +1217,9 @@ func (t *freezerTable) advanceHead() error { return err } t.releaseFile(t.headId) - t.openFile(t.headId, openFreezerFileForReadOnly) + if _, err := t.openFile(t.headId, openFreezerFileForReadOnly); err != nil { + return err + } // Swap out the current head. t.head = newHead diff --git a/core/state/access_events.go b/core/state/access_events.go index 0575c9898a..86f44bd623 100644 --- a/core/state/access_events.go +++ b/core/state/access_events.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/holiman/uint256" ) @@ -45,15 +45,12 @@ var zeroTreeIndex uint256.Int type AccessEvents struct { branches map[branchAccessKey]mode chunks map[chunkAccessKey]mode - - pointCache *utils.PointCache } -func NewAccessEvents(pointCache *utils.PointCache) *AccessEvents { +func NewAccessEvents() *AccessEvents { return &AccessEvents{ - branches: make(map[branchAccessKey]mode), - chunks: make(map[chunkAccessKey]mode), - pointCache: pointCache, + branches: make(map[branchAccessKey]mode), + chunks: make(map[chunkAccessKey]mode), } } @@ -75,8 +72,11 @@ func (ae *AccessEvents) Keys() [][]byte { // TODO: consider if parallelizing this is worth it, probably depending on len(ae.chunks). keys := make([][]byte, 0, len(ae.chunks)) for chunk := range ae.chunks { - basePoint := ae.pointCache.Get(chunk.addr[:]) - key := utils.GetTreeKeyWithEvaluatedAddress(basePoint, &chunk.treeIndex, chunk.leafKey) + var offset [32]byte + treeIndexBytes := chunk.treeIndex.Bytes32() + copy(offset[:31], treeIndexBytes[1:]) + offset[31] = chunk.leafKey + key := bintrie.GetBinaryTreeKey(chunk.addr, offset[:]) keys = append(keys, key) } return keys @@ -84,9 +84,8 @@ func (ae *AccessEvents) Keys() [][]byte { func (ae *AccessEvents) Copy() *AccessEvents { cpy := &AccessEvents{ - branches: maps.Clone(ae.branches), - chunks: maps.Clone(ae.chunks), - pointCache: ae.pointCache, + branches: maps.Clone(ae.branches), + chunks: maps.Clone(ae.chunks), } return cpy } @@ -95,12 +94,12 @@ func (ae *AccessEvents) Copy() *AccessEvents { // member fields of an account. func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableGas uint64) uint64 { var gas uint64 // accumulate the consumed gas - consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if consumed < expected { return expected } gas += consumed - consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas-consumed) + consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas-consumed) if consumed < expected { return expected + gas } @@ -112,7 +111,7 @@ func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableG // cold member fields of an account, that need to be touched when making a message // call to that account. func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas uint64) uint64 { - _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) + _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) if expected == 0 { expected = params.WarmStorageReadCostEIP2929 } @@ -122,11 +121,11 @@ func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas // ValueTransferGas returns the gas to be charged for each of the currently // cold balance member fields of the caller and the callee accounts. func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, availableGas uint64) uint64 { - _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) if expected1 > availableGas { return expected1 } - _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas-expected1) + _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas-expected1) if expected1+expected2 == 0 { return params.WarmStorageReadCostEIP2929 } @@ -138,8 +137,8 @@ func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, // address collision is done before the transfer, and so no write // are guaranteed to happen at this point. func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, availableGas uint64) uint64 { - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) - _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, false, availableGas-consumed) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) + _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, availableGas-consumed) return expected1 + expected2 } @@ -147,9 +146,9 @@ func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, available // a contract creation. func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas uint64) (uint64, uint64) { var gas uint64 - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) gas += consumed - consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, true, availableGas-consumed) + consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, true, availableGas-consumed) gas += consumed return gas, expected1 + expected2 } @@ -157,20 +156,20 @@ func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas // AddTxOrigin adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxOrigin(originAddr common.Address) { - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, gomath.MaxUint64) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeHashLeafKey, false, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, gomath.MaxUint64) } // AddTxDestination adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxDestination(addr common.Address, sendsValue, doesntExist bool) { - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, sendsValue, gomath.MaxUint64) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, doesntExist, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, sendsValue, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, doesntExist, gomath.MaxUint64) } // SlotGas returns the amount of gas to be charged for a cold storage access. func (ae *AccessEvents) SlotGas(addr common.Address, slot common.Hash, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - treeIndex, subIndex := utils.StorageIndex(slot.Bytes()) + treeIndex, subIndex := bintrie.StorageIndex(slot.Bytes()) _, expected := ae.touchAddressAndChargeGas(addr, *treeIndex, subIndex, isWrite, availableGas) if expected == 0 && chargeWarmCosts { expected = params.WarmStorageReadCostEIP2929 @@ -313,7 +312,7 @@ func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, // Note that an access in write mode implies an access in read mode, whereas an // access in read mode does not imply an access in write mode. func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas @@ -329,7 +328,7 @@ func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availabl // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) CodeHashGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas diff --git a/core/state/access_events_test.go b/core/state/access_events_test.go index e80859a0b4..0b39130e8d 100644 --- a/core/state/access_events_test.go +++ b/core/state/access_events_test.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) var ( @@ -38,7 +37,7 @@ func init() { } func TestAccountHeaderGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost gas := ae.BasicDataGas(testAddr, false, math.MaxUint64, false) @@ -93,7 +92,7 @@ func TestAccountHeaderGas(t *testing.T) { // TestContractCreateInitGas checks that the gas cost of contract creation is correctly // calculated. func TestContractCreateInitGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() var testAddr [20]byte for i := byte(0); i < 20; i++ { @@ -116,7 +115,7 @@ func TestContractCreateInitGas(t *testing.T) { // TestMessageCallGas checks that the gas cost of message calls is correctly // calculated. func TestMessageCallGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost, without a value gas := ae.MessageCallGas(testAddr, math.MaxUint64) diff --git a/core/state/database.go b/core/state/database.go index e1236c6571..df936ba928 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -28,8 +28,9 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/ethereum/go-ethereum/triedb" ) @@ -42,9 +43,6 @@ const ( // Cache size granted for caching clean code. codeCacheSize = 256 * 1024 * 1024 - - // Number of address->curve point associations to keep. - pointCacheSize = 4096 ) // Database wraps access to tries and contract code. @@ -65,9 +63,6 @@ type Database interface { // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore - // PointCache returns the cache holding points used in verkle tree key computation - PointCache() *utils.PointCache - // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *triedb.Database @@ -178,7 +173,6 @@ type CachingDB struct { snap *snapshot.Tree codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache // Transition-specific fields TransitionStatePerRoot *lru.Cache[common.Hash, *overlay.TransitionState] @@ -197,7 +191,6 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { snap: snap, codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), } } @@ -208,8 +201,8 @@ func NewDatabaseForTesting() *CachingDB { return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil) } -// Reader returns a state reader associated with the specified state root. -func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { +// StateReader returns a state reader associated with the specified state root. +func (db *CachingDB) StateReader(stateRoot common.Hash) (StateReader, error) { var readers []StateReader // Configure the state reader using the standalone snapshot in hash mode. @@ -233,29 +226,38 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } // Configure the trie reader, which is expected to be available as the // gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb) if err != nil { return nil, err } readers = append(readers, tr) - combined, err := newMultiStateReader(readers...) + return newMultiStateReader(readers...) +} + +// Reader implements Database, returning a reader associated with the specified +// state root. +func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { + sr, err := db.StateReader(stateRoot) if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil + return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), sr), nil } -// ReadersWithCacheStats creates a pair of state readers sharing the same internal cache and -// same backing Reader, but exposing separate statistics. -// and statistics. +// ReadersWithCacheStats creates a pair of state readers that share the same +// underlying state reader and internal state cache, while maintaining separate +// statistics respectively. func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) { - reader, err := db.Reader(stateRoot) + r, err := db.StateReader(stateRoot) if err != nil { return nil, nil, err } - shared := newReaderWithCache(reader) - return newReaderWithCacheStats(shared), newReaderWithCacheStats(shared), nil + sr := newStateReaderWithCache(r) + + ra := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) + rb := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) + return ra, rb, nil } // OpenTrie opens the main account trie at a specific root hash. @@ -263,10 +265,12 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.triedb.IsVerkle() { ts := overlay.LoadTransitionState(db.TrieDB().Disk(), root, db.triedb.IsVerkle()) if ts.InTransition() { - panic("transition isn't supported yet") + panic("state tree transition isn't supported yet") } if ts.Transitioned() { - return trie.NewVerkleTrie(root, db.triedb, db.pointCache) + // Use BinaryTrie instead of VerkleTrie when IsVerkle is set + // (IsVerkle actually means Binary Trie mode in this codebase) + return bintrie.NewBinaryTrie(root, db.triedb) } } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) @@ -309,11 +313,6 @@ func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb } -// PointCache returns the cache of evaluated curve points. -func (db *CachingDB) PointCache() *utils.PointCache { - return db.pointCache -} - // Snapshot returns the underlying state snapshot. func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap @@ -324,9 +323,7 @@ func mustCopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.StateTrie: return t.Copy() - case *trie.VerkleTrie: - return t.Copy() - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/database_history.go b/core/state/database_history.go index b2fda19e4f..81d2d4f65e 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -17,42 +17,40 @@ package state import ( - "errors" + "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" "github.com/ethereum/go-ethereum/triedb/pathdb" ) -// historicReader wraps a historical state reader defined in path database, -// providing historic state serving over the path scheme. -// -// TODO(rjl493456442): historicReader is not thread-safe and does not fully -// comply with the StateReader interface requirements, needs to be fixed. -// Currently, it is only used in a non-concurrent context, so it is safe for now. -type historicReader struct { +// historicStateReader implements StateReader, wrapping a historical state reader +// defined in path database and provide historic state serving over the path scheme. +type historicStateReader struct { reader *pathdb.HistoricalStateReader + lock sync.Mutex // Lock for protecting concurrent read } -// newHistoricReader constructs a reader for historic state serving. -func newHistoricReader(r *pathdb.HistoricalStateReader) *historicReader { - return &historicReader{reader: r} +// newHistoricStateReader constructs a reader for historical state serving. +func newHistoricStateReader(r *pathdb.HistoricalStateReader) *historicStateReader { + return &historicStateReader{reader: r} } // Account implements StateReader, retrieving the account specified by the address. -// -// An error will be returned if the associated snapshot is already stale or -// the requested account is not yet covered by the snapshot. -// -// The returned account might be nil if it's not existent. -func (r *historicReader) Account(addr common.Address) (*types.StateAccount, error) { +func (r *historicStateReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + account, err := r.reader.Account(addr) if err != nil { return nil, err @@ -82,7 +80,10 @@ func (r *historicReader) Account(addr common.Address) (*types.StateAccount, erro // the requested storage slot is not yet covered by the snapshot. // // The returned storage slot might be empty if it's not existent. -func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { +func (r *historicStateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + blob, err := r.reader.Storage(addr, key) if err != nil { return common.Hash{}, err @@ -99,6 +100,125 @@ func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.H return slot, nil } +// historicTrieOpener is a wrapper of pathdb.HistoricalNodeReader, implementing +// the database.NodeDatabase by adding NodeReader function. +type historicTrieOpener struct { + root common.Hash + reader *pathdb.HistoricalNodeReader +} + +// newHistoricTrieOpener constructs the historical trie opener. +func newHistoricTrieOpener(root common.Hash, reader *pathdb.HistoricalNodeReader) *historicTrieOpener { + return &historicTrieOpener{ + root: root, + reader: reader, + } +} + +// NodeReader implements database.NodeDatabase, returning a node reader of a +// specified state. +func (o *historicTrieOpener) NodeReader(root common.Hash) (database.NodeReader, error) { + if root != o.root { + return nil, fmt.Errorf("state %x is not available", root) + } + return o.reader, nil +} + +// historicalTrieReader wraps a historical node reader defined in path database, +// providing historical node serving over the path scheme. +type historicalTrieReader struct { + root common.Hash + opener *historicTrieOpener + tr Trie + + subRoots map[common.Address]common.Hash // Set of storage roots, cached when the account is resolved + subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved + lock sync.Mutex // Lock for protecting concurrent read +} + +// newHistoricalTrieReader constructs a reader for historical trie node serving. +func newHistoricalTrieReader(root common.Hash, r *pathdb.HistoricalNodeReader) (*historicalTrieReader, error) { + opener := newHistoricTrieOpener(root, r) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), opener) + if err != nil { + return nil, err + } + return &historicalTrieReader{ + root: root, + opener: opener, + tr: tr, + subRoots: make(map[common.Address]common.Hash), + subTries: make(map[common.Address]Trie), + }, nil +} + +// account is the inner version of Account and assumes the r.lock is already held. +func (r *historicalTrieReader) account(addr common.Address) (*types.StateAccount, error) { + account, err := r.tr.GetAccount(addr) + if err != nil { + return nil, err + } + if account == nil { + r.subRoots[addr] = types.EmptyRootHash + } else { + r.subRoots[addr] = account.Root + } + return account, nil +} + +// Account implements StateReader, retrieving the account specified by the address. +// +// An error will be returned if the associated snapshot is already stale or +// the requested account is not yet covered by the snapshot. +// +// The returned account might be nil if it's not existent. +func (r *historicalTrieReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + + return r.account(addr) +} + +// Storage implements StateReader, retrieving the storage slot specified by the +// address and slot key. +// +// An error will be returned if the associated snapshot is already stale or +// the requested storage slot is not yet covered by the snapshot. +// +// The returned storage slot might be empty if it's not existent. +func (r *historicalTrieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + + tr, found := r.subTries[addr] + if !found { + root, ok := r.subRoots[addr] + + // The storage slot is accessed without account caching. It's unexpected + // behavior but try to resolve the account first anyway. + if !ok { + _, err := r.account(addr) + if err != nil { + return common.Hash{}, err + } + root = r.subRoots[addr] + } + var err error + tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.Keccak256Hash(addr.Bytes()), root), r.opener) + if err != nil { + return common.Hash{}, err + } + r.subTries[addr] = tr + } + ret, err := tr.GetStorage(addr, key.Bytes()) + if err != nil { + return common.Hash{}, err + } + var value common.Hash + value.SetBytes(ret) + return value, nil +} + // HistoricDB is the implementation of Database interface, with the ability to // access historical state. type HistoricDB struct { @@ -107,7 +227,6 @@ type HistoricDB struct { triedb *triedb.Database codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache } // NewHistoricDatabase creates a historic state database. @@ -119,33 +238,59 @@ func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *His triedb: triedb, codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), } } // Reader implements Database interface, returning a reader of the specific state. func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) { - hr, err := db.triedb.HistoricReader(stateRoot) + var readers []StateReader + sr, err := db.triedb.HistoricStateReader(stateRoot) + if err == nil { + readers = append(readers, newHistoricStateReader(sr)) + } + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err == nil { + tr, err := newHistoricalTrieReader(stateRoot, nr) + if err == nil { + readers = append(readers, tr) + } + } + if len(readers) == 0 { + return nil, fmt.Errorf("historical state %x is not available", stateRoot) + } + combined, err := newMultiStateReader(readers...) if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), newHistoricReader(hr)), nil + return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil } // OpenTrie opens the main account trie. It's not supported by historic database. func (db *HistoricDB) OpenTrie(root common.Hash) (Trie, error) { - return nil, errors.New("not implemented") + nr, err := db.triedb.HistoricNodeReader(root) + if err != nil { + return nil, err + } + tr, err := trie.NewStateTrie(trie.StateTrieID(root), newHistoricTrieOpener(root, nr)) + if err != nil { + return nil, err + } + return tr, nil } // OpenStorageTrie opens the storage trie of an account. It's not supported by // historic database. -func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) { - return nil, errors.New("not implemented") -} - -// PointCache returns the cache holding points used in verkle tree key computation -func (db *HistoricDB) PointCache() *utils.PointCache { - return db.pointCache +func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) { + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err != nil { + return nil, err + } + id := trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root) + tr, err := trie.NewStateTrie(id, newHistoricTrieOpener(stateRoot, nr)) + if err != nil { + return nil, err + } + return tr, nil } // TrieDB returns the underlying trie database for managing trie nodes. diff --git a/core/state/dump.go b/core/state/dump.go index a4abc33733..829d106ed3 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -18,6 +18,7 @@ package state import ( "encoding/json" + "errors" "fmt" "time" @@ -27,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" ) // DumpConfig is a set of options to control what portions of the state will be @@ -221,6 +223,28 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] return nextKey } +// DumpBinTrieLeaves collects all binary trie leaf nodes into the provided map. +func (s *StateDB) DumpBinTrieLeaves(collector map[common.Hash]hexutil.Bytes) error { + tr, err := s.db.OpenTrie(s.originalRoot) + if err != nil { + return err + } + btr, ok := tr.(*bintrie.BinaryTrie) + if !ok { + return errors.New("trie is not a binary trie") + } + it, err := btr.NodeIterator(nil) + if err != nil { + return err + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + return nil +} + // RawDump returns the state. If the processing is aborted e.g. due to options // reaching Max, the `Next` key is set on the returned Dump. func (s *StateDB) RawDump(opts *DumpConfig) Dump { diff --git a/core/state/journal.go b/core/state/journal.go index 28af9317f3..89877662f9 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -43,7 +43,8 @@ type journalEntry interface { revert(*StateDB) // dirtied returns the Ethereum address modified by this journal entry. - dirtied() *common.Address + // indicates false if no address was changed. + dirtied() (common.Address, bool) // copy returns a deep-copied journal entry. copy() journalEntry @@ -110,11 +111,11 @@ func (j *journal) revertToSnapshot(revid int, s *StateDB) { // append inserts a new modification entry to the end of the change journal. func (j *journal) append(entry journalEntry) { j.entries = append(j.entries, entry) - if addr := entry.dirtied(); addr != nil { - j.dirties[*addr]++ + if addr, dirty := entry.dirtied(); dirty { + j.dirties[addr]++ // Arbitrum: also track the number of zombie changes if isZombie(entry) { - j.zombieEntries[*addr]++ + j.zombieEntries[addr]++ } } } @@ -127,14 +128,14 @@ func (j *journal) revert(statedb *StateDB, snapshot int) { j.entries[i].revert(statedb) // Drop any dirty tracking induced by the change - if addr := j.entries[i].dirtied(); addr != nil { - if j.dirties[*addr]--; j.dirties[*addr] == 0 { - delete(j.dirties, *addr) + if addr, dirty := j.entries[i].dirtied(); dirty { + if j.dirties[addr]--; j.dirties[addr] == 0 { + delete(j.dirties, addr) // Revert zombieEntries tracking if isZombie(j.entries[i]) { - if j.zombieEntries[*addr]--; j.zombieEntries[*addr] == 0 { - delete(j.zombieEntries, *addr) + if j.zombieEntries[addr]--; j.zombieEntries[addr] == 0 { + delete(j.zombieEntries, addr) } } } @@ -323,8 +324,8 @@ func (ch createObjectChange) revert(s *StateDB) { delete(s.stateObjects, ch.account) } -func (ch createObjectChange) dirtied() *common.Address { - return &ch.account +func (ch createObjectChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch createObjectChange) copy() journalEntry { @@ -337,8 +338,8 @@ func (ch createContractChange) revert(s *StateDB) { s.getStateObject(ch.account).newContract = false } -func (ch createContractChange) dirtied() *common.Address { - return nil +func (ch createContractChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch createContractChange) copy() journalEntry { @@ -354,8 +355,8 @@ func (ch selfDestructChange) revert(s *StateDB) { } } -func (ch selfDestructChange) dirtied() *common.Address { - return &ch.account +func (ch selfDestructChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch selfDestructChange) copy() journalEntry { @@ -369,8 +370,8 @@ var ripemd = common.HexToAddress("0000000000000000000000000000000000000003") func (ch touchChange) revert(s *StateDB) { } -func (ch touchChange) dirtied() *common.Address { - return &ch.account +func (ch touchChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch touchChange) copy() journalEntry { @@ -383,8 +384,8 @@ func (ch balanceChange) revert(s *StateDB) { s.getStateObject(ch.account).setBalance(ch.prev) } -func (ch balanceChange) dirtied() *common.Address { - return &ch.account +func (ch balanceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch balanceChange) copy() journalEntry { @@ -398,8 +399,8 @@ func (ch nonceChange) revert(s *StateDB) { s.getStateObject(ch.account).setNonce(ch.prev) } -func (ch nonceChange) dirtied() *common.Address { - return &ch.account +func (ch nonceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch nonceChange) copy() journalEntry { @@ -413,8 +414,8 @@ func (ch codeChange) revert(s *StateDB) { s.getStateObject(ch.account).setCode(crypto.Keccak256Hash(ch.prevCode), ch.prevCode) } -func (ch codeChange) dirtied() *common.Address { - return &ch.account +func (ch codeChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch codeChange) copy() journalEntry { @@ -428,8 +429,8 @@ func (ch storageChange) revert(s *StateDB) { s.getStateObject(ch.account).setState(ch.key, ch.prevvalue, ch.origvalue) } -func (ch storageChange) dirtied() *common.Address { - return &ch.account +func (ch storageChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch storageChange) copy() journalEntry { @@ -445,8 +446,8 @@ func (ch transientStorageChange) revert(s *StateDB) { s.setTransientState(ch.account, ch.key, ch.prevalue) } -func (ch transientStorageChange) dirtied() *common.Address { - return nil +func (ch transientStorageChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch transientStorageChange) copy() journalEntry { @@ -461,8 +462,8 @@ func (ch refundChange) revert(s *StateDB) { s.refund = ch.prev } -func (ch refundChange) dirtied() *common.Address { - return nil +func (ch refundChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch refundChange) copy() journalEntry { @@ -481,8 +482,8 @@ func (ch addLogChange) revert(s *StateDB) { s.logSize-- } -func (ch addLogChange) dirtied() *common.Address { - return nil +func (ch addLogChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch addLogChange) copy() journalEntry { @@ -504,8 +505,8 @@ func (ch accessListAddAccountChange) revert(s *StateDB) { s.accessList.DeleteAddress(ch.address) } -func (ch accessListAddAccountChange) dirtied() *common.Address { - return nil +func (ch accessListAddAccountChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddAccountChange) copy() journalEntry { @@ -518,8 +519,8 @@ func (ch accessListAddSlotChange) revert(s *StateDB) { s.accessList.DeleteSlot(ch.address, ch.slot) } -func (ch accessListAddSlotChange) dirtied() *common.Address { - return nil +func (ch accessListAddSlotChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddSlotChange) copy() journalEntry { diff --git a/core/state/journal_arbitrum.go b/core/state/journal_arbitrum.go index 1561b4304e..039d4701fd 100644 --- a/core/state/journal_arbitrum.go +++ b/core/state/journal_arbitrum.go @@ -13,8 +13,8 @@ func (ch wasmActivation) revert(s *StateDB) { delete(s.arbExtraData.activatedWasms, ch.moduleHash) } -func (ch wasmActivation) dirtied() *common.Address { - return nil +func (ch wasmActivation) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch wasmActivation) copy() journalEntry { @@ -38,8 +38,8 @@ func (ch CacheWasm) revert(s *StateDB) { EvictWasmRust(ch.ModuleHash, ch.Version, ch.Tag, ch.Debug) } -func (ch CacheWasm) dirtied() *common.Address { - return nil +func (ch CacheWasm) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch CacheWasm) copy() journalEntry { @@ -67,8 +67,8 @@ func (ch EvictWasm) revert(s *StateDB) { } } -func (ch EvictWasm) dirtied() *common.Address { - return nil +func (ch EvictWasm) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch EvictWasm) copy() journalEntry { @@ -96,8 +96,11 @@ func (ch createZombieChange) revert(s *StateDB) { delete(s.stateObjects, *ch.account) } -func (ch createZombieChange) dirtied() *common.Address { - return ch.account +func (ch createZombieChange) dirtied() (common.Address, bool) { + if ch.account == nil { + return common.Address{}, false + } + return *ch.account, true } func (ch createZombieChange) copy() journalEntry { diff --git a/core/state/reader.go b/core/state/reader.go index 3e8b31b6be..35b732173b 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -18,6 +18,7 @@ package state import ( "errors" + "fmt" "sync" "sync/atomic" @@ -30,13 +31,18 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/database" ) // ContractCodeReader defines the interface for accessing contract code. type ContractCodeReader interface { + // Has returns the flag indicating whether the contract code with + // specified address and hash exists or not. + Has(addr common.Address, codeHash common.Hash) bool + // Code retrieves a particular contract's code. // // - Returns nil code along with nil error if the requested contract code @@ -52,6 +58,30 @@ type ContractCodeReader interface { CodeSize(addr common.Address, codeHash common.Hash) (int, error) } +// ContractCodeReaderStats aggregates statistics for the contract code reader. +type ContractCodeReaderStats struct { + CacheHit int64 // Number of cache hits + CacheMiss int64 // Number of cache misses + CacheHitBytes int64 // Total bytes served from cache + CacheMissBytes int64 // Total bytes read on cache misses +} + +// HitRate returns the cache hit rate. +func (s ContractCodeReaderStats) HitRate() float64 { + if s.CacheHit == 0 { + return 0 + } + return float64(s.CacheHit) / float64(s.CacheHit+s.CacheMiss) +} + +// ContractCodeReaderWithStats extends ContractCodeReader by adding GetStats to +// expose statistics of code reader. +type ContractCodeReaderWithStats interface { + ContractCodeReader + + GetStats() ContractCodeReaderStats +} + // StateReader defines the interface for accessing accounts and storage slots // associated with a specific state. // @@ -86,10 +116,30 @@ type Reader interface { // ReaderStats wraps the statistics of reader. type ReaderStats struct { - AccountHit int64 - AccountMiss int64 - StorageHit int64 - StorageMiss int64 + AccountCacheHit int64 + AccountCacheMiss int64 + StorageCacheHit int64 + StorageCacheMiss int64 + CodeStats ContractCodeReaderStats +} + +// String implements fmt.Stringer, returning string format statistics. +func (s ReaderStats) String() string { + var ( + accountCacheHitRate float64 + storageCacheHitRate float64 + ) + if s.AccountCacheHit > 0 { + accountCacheHitRate = float64(s.AccountCacheHit) / float64(s.AccountCacheHit+s.AccountCacheMiss) * 100 + } + if s.StorageCacheHit > 0 { + storageCacheHitRate = float64(s.StorageCacheHit) / float64(s.StorageCacheHit+s.StorageCacheMiss) * 100 + } + msg := fmt.Sprintf("Reader statistics\n") + msg += fmt.Sprintf("account: hit: %d, miss: %d, rate: %.2f\n", s.AccountCacheHit, s.AccountCacheMiss, accountCacheHitRate) + msg += fmt.Sprintf("storage: hit: %d, miss: %d, rate: %.2f\n", s.StorageCacheHit, s.StorageCacheMiss, storageCacheHitRate) + msg += fmt.Sprintf("code: hit: %d(%v), miss: %d(%v), rate: %.2f\n", s.CodeStats.CacheHit, common.StorageSize(s.CodeStats.CacheHitBytes), s.CodeStats.CacheMiss, common.StorageSize(s.CodeStats.CacheMissBytes), s.CodeStats.HitRate()) + return msg } // ReaderWithStats wraps the additional method to retrieve the reader statistics from. @@ -109,6 +159,12 @@ type cachingCodeReader struct { // they are natively thread-safe. codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] + + // Cache statistics + hit atomic.Int64 // Number of code lookups found in the cache + miss atomic.Int64 // Number of code lookups not found in the cache + hitBytes atomic.Int64 // Total number of bytes read from cache + missBytes atomic.Int64 // Total number of bytes read from database } // newCachingCodeReader constructs the code reader. @@ -125,12 +181,17 @@ func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstraine func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) { code, _ := r.codeCache.Get(codeHash) if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) return code, nil } + r.miss.Add(1) + code = rawdb.ReadCode(r.db, codeHash) if len(code) > 0 { r.codeCache.Add(codeHash, code) r.codeSizeCache.Add(codeHash, len(code)) + r.missBytes.Add(int64(len(code))) } return code, nil } @@ -139,6 +200,7 @@ func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]b // If the contract code doesn't exist, no error will be returned. func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) { if cached, ok := r.codeSizeCache.Get(codeHash); ok { + r.hit.Add(1) return cached, nil } code, err := r.Code(addr, codeHash) @@ -148,6 +210,23 @@ func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) return len(code), nil } +// Has returns the flag indicating whether the contract code with +// specified address and hash exists or not. +func (r *cachingCodeReader) Has(addr common.Address, codeHash common.Hash) bool { + code, _ := r.Code(addr, codeHash) + return len(code) > 0 +} + +// GetStats returns the statistics of the code reader. +func (r *cachingCodeReader) GetStats() ContractCodeReaderStats { + return ContractCodeReaderStats{ + CacheHit: r.hit.Load(), + CacheMiss: r.miss.Load(), + CacheHitBytes: r.hitBytes.Load(), + CacheMissBytes: r.missBytes.Load(), + } +} + // flatReader wraps a database state reader and is safe for concurrent access. type flatReader struct { reader database.StateReader @@ -234,7 +313,7 @@ type trieReader struct { // newTrieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database) (*trieReader, error) { var ( tr Trie err error @@ -242,7 +321,11 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if !db.IsVerkle() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { - tr, err = trie.NewVerkleTrie(root, db, cache) + // When IsVerkle() is true, create a BinaryTrie wrapped in TransitionTrie + binTrie, binErr := bintrie.NewBinaryTrie(root, db) + if binErr != nil { + return nil, binErr + } // Based on the transition status, determine if the overlay // tree needs to be created, or if a single, target tree is @@ -253,7 +336,22 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if err != nil { return nil, err } - tr = trie.NewTransitionTrie(mpt, tr.(*trie.VerkleTrie), false) + tr = transitiontrie.NewTransitionTrie(mpt, binTrie, false) + } else { + // HACK: Use TransitionTrie with nil base as a wrapper to make BinaryTrie + // satisfy the Trie interface. This works around the import cycle between + // trie and trie/bintrie packages. + // + // TODO: In future PRs, refactor the package structure to avoid this hack: + // - Option 1: Move common interfaces (Trie, NodeIterator) to a separate + // package that both trie and trie/bintrie can import + // - Option 2: Create a factory function in the trie package that returns + // BinaryTrie as a Trie interface without direct import + // - Option 3: Move BinaryTrie to the main trie package + // + // The current approach works but adds unnecessary overhead and complexity + // by using TransitionTrie when there's no actual transition happening. + tr = transitiontrie.NewTransitionTrie(nil, binTrie, false) } } if err != nil { @@ -411,10 +509,10 @@ func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { } } -// readerWithCache is a wrapper around Reader that maintains additional state caches -// to support concurrent state access. -type readerWithCache struct { - Reader // safe for concurrent read +// stateReaderWithCache is a wrapper around StateReader that maintains additional +// state caches to support concurrent state access. +type stateReaderWithCache struct { + StateReader // Previously resolved state entries. accounts map[common.Address]*types.StateAccount @@ -430,11 +528,11 @@ type readerWithCache struct { } } -// newReaderWithCache constructs the reader with local cache. -func newReaderWithCache(reader Reader) *readerWithCache { - r := &readerWithCache{ - Reader: reader, - accounts: make(map[common.Address]*types.StateAccount), +// newStateReaderWithCache constructs the state reader with local cache. +func newStateReaderWithCache(sr StateReader) *stateReaderWithCache { + r := &stateReaderWithCache{ + StateReader: sr, + accounts: make(map[common.Address]*types.StateAccount), } for i := range r.storageBuckets { r.storageBuckets[i].storages = make(map[common.Address]map[common.Hash]common.Hash) @@ -447,7 +545,7 @@ func newReaderWithCache(reader Reader) *readerWithCache { // might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, bool, error) { +func (r *stateReaderWithCache) account(addr common.Address) (*types.StateAccount, bool, error) { // Try to resolve the requested account in the local cache r.accountLock.RLock() acct, ok := r.accounts[addr] @@ -456,7 +554,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo return acct, true, nil } // Try to resolve the requested account from the underlying reader - acct, err := r.Reader.Account(addr) + acct, err := r.StateReader.Account(addr) if err != nil { return nil, false, err } @@ -470,7 +568,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, error) { +func (r *stateReaderWithCache) Account(addr common.Address) (*types.StateAccount, error) { account, _, err := r.account(addr) return account, err } @@ -478,7 +576,7 @@ func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, err // storage retrieves the storage slot specified by the address and slot key, along // with a flag indicating whether it's found in the cache or not. The returned // storage slot might be empty if it's not existent. -func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) { +func (r *stateReaderWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) { var ( value common.Hash ok bool @@ -495,7 +593,7 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common return value, true, nil } // Try to resolve the requested storage slot from the underlying reader - value, err := r.Reader.Storage(addr, slot) + value, err := r.StateReader.Storage(addr, slot) if err != nil { return common.Hash{}, false, err } @@ -516,23 +614,26 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { +func (r *stateReaderWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { value, _, err := r.storage(addr, slot) return value, err } -type readerWithCacheStats struct { - *readerWithCache - accountHit atomic.Int64 - accountMiss atomic.Int64 - storageHit atomic.Int64 - storageMiss atomic.Int64 +type readerWithStats struct { + *stateReaderWithCache + ContractCodeReaderWithStats + + accountCacheHit atomic.Int64 + accountCacheMiss atomic.Int64 + storageCacheHit atomic.Int64 + storageCacheMiss atomic.Int64 } -// newReaderWithCacheStats constructs the reader with additional statistics tracked. -func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats { - return &readerWithCacheStats{ - readerWithCache: reader, +// newReaderWithStats constructs the reader with additional statistics tracked. +func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats) *readerWithStats { + return &readerWithStats{ + stateReaderWithCache: sr, + ContractCodeReaderWithStats: cr, } } @@ -540,15 +641,15 @@ func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats { // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount, error) { - account, incache, err := r.readerWithCache.account(addr) +func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, error) { + account, incache, err := r.stateReaderWithCache.account(addr) if err != nil { return nil, err } if incache { - r.accountHit.Add(1) + r.accountCacheHit.Add(1) } else { - r.accountMiss.Add(1) + r.accountCacheMiss.Add(1) } return account, nil } @@ -558,25 +659,26 @@ func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithCacheStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { - value, incache, err := r.readerWithCache.storage(addr, slot) +func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { + value, incache, err := r.stateReaderWithCache.storage(addr, slot) if err != nil { return common.Hash{}, err } if incache { - r.storageHit.Add(1) + r.storageCacheHit.Add(1) } else { - r.storageMiss.Add(1) + r.storageCacheMiss.Add(1) } return value, nil } // GetStats implements ReaderWithStats, returning the statistics of state reader. -func (r *readerWithCacheStats) GetStats() ReaderStats { +func (r *readerWithStats) GetStats() ReaderStats { return ReaderStats{ - AccountHit: r.accountHit.Load(), - AccountMiss: r.accountMiss.Load(), - StorageHit: r.storageHit.Load(), - StorageMiss: r.storageMiss.Load(), + AccountCacheHit: r.accountCacheHit.Load(), + AccountCacheMiss: r.accountCacheMiss.Load(), + StorageCacheHit: r.storageCacheHit.Load(), + StorageCacheMiss: r.storageCacheMiss.Load(), + CodeStats: r.ContractCodeReaderWithStats.GetStats(), } } diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 28957051d4..1286ded7e1 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -465,6 +465,6 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { storageList := slices.SortedFunc(maps.Keys(dl.storageData[accountHash]), common.Hash.Cmp) dl.storageList[accountHash] = storageList - dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) + dl.memory += uint64(len(storageList)*common.HashLength + common.HashLength) return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 2c868b3010..90a265645d 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -198,6 +198,39 @@ func TestInsertAndMerge(t *testing.T) { } } +// TestStorageListMemoryAccounting ensures that StorageList increases dl.memory +// proportionally to the number of storage slots in the requested account and +// does not change memory usage on repeated calls for the same account. +func TestStorageListMemoryAccounting(t *testing.T) { + parent := newDiffLayer(emptyLayer(), common.Hash{}, nil, nil) + account := common.HexToHash("0x01") + + slots := make(map[common.Hash][]byte) + for i := 0; i < 3; i++ { + slots[randomHash()] = []byte{0x01} + } + storage := map[common.Hash]map[common.Hash][]byte{ + account: slots, + } + dl := newDiffLayer(parent, common.Hash{}, nil, storage) + + before := dl.memory + list := dl.StorageList(account) + if have, want := len(list), len(slots); have != want { + t.Fatalf("StorageList length mismatch: have %d, want %d", have, want) + } + expectedDelta := uint64(len(list)*common.HashLength + common.HashLength) + if have, want := dl.memory-before, expectedDelta; have != want { + t.Fatalf("StorageList memory delta mismatch: have %d, want %d", have, want) + } + + before = dl.memory + _ = dl.StorageList(account) + if dl.memory != before { + t.Fatalf("StorageList changed memory on cached call: have %d, want %d", dl.memory, before) + } +} + func emptyLayer() *diskLayer { return &diskLayer{ diskdb: memorydb.New(), diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 644e624ea5..f0b13d4ba7 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/arbcrypto" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -38,12 +38,7 @@ import ( ) func hashData(input []byte) common.Hash { - var hasher = arbcrypto.NewLegacyKeccak256() - var hash common.Hash - hasher.Reset() - hasher.Write(input) - hasher.Sum(hash[:0]) - return hash + return crypto.Keccak256Hash(input) } // Tests that snapshot generation from an empty database. diff --git a/core/state/state_object.go b/core/state/state_object.go index 28ffcac735..9451a8c18e 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" ) @@ -47,11 +49,11 @@ func (s Storage) Copy() Storage { // - Account values as well as storages can be accessed and modified through the object. // - Finally, call commit to return the changes of storage trie and update account data. type stateObject struct { - db *StateDB - address common.Address // address of ethereum account - addrHash common.Hash // hash of ethereum address of the account - origin *types.StateAccount // Account original data without any change applied, nil means it was not existent - data types.StateAccount // Account data with all mutations applied in the scope of block + db *StateDB + address common.Address // address of ethereum account + addressHash *common.Hash // hash of ethereum address of the account + origin *types.StateAccount // Account original data without any change applied, nil means it was not existent + data types.StateAccount // Account data with all mutations applied in the scope of block // Write caches. trie Trie // storage trie, which becomes non-nil on first access @@ -101,7 +103,6 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s return &stateObject{ db: db, address: address, - addrHash: crypto.Keccak256Hash(address[:]), origin: origin, data: *acct, originStorage: make(Storage), @@ -111,6 +112,14 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s } } +func (s *stateObject) addrHash() common.Hash { + if s.addressHash == nil { + h := crypto.Keccak256Hash(s.address[:]) + s.addressHash = &h + } + return *s.addressHash +} + func (s *stateObject) markSelfdestructed() { s.selfDestructed = true } @@ -150,7 +159,7 @@ func (s *stateObject) getPrefetchedTrie() Trie { return nil } // Attempt to retrieve the trie from the prefetcher - return s.db.prefetcher.trie(s.addrHash, s.data.Root) + return s.db.prefetcher.trie(s.addrHash(), s.data.Root) } // GetState retrieves a value associated with the given storage key. @@ -202,7 +211,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // Schedule the resolved storage slots for prefetching if it's enabled. if s.db.prefetcher != nil && s.data.Root != types.EmptyRootHash { - if err = s.db.prefetcher.prefetch(s.addrHash, s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { + if err = s.db.prefetcher.prefetch(s.addrHash(), s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { log.Error("Failed to prefetch storage slot", "addr", s.address, "key", key, "err", err) } } @@ -263,7 +272,7 @@ func (s *stateObject) finalise() { s.pendingStorage[key] = value } if s.db.prefetcher != nil && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { - if err := s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { + if err := s.db.prefetcher.prefetch(s.addrHash(), s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { log.Error("Failed to prefetch slots", "addr", s.address, "slots", len(slotsToPrefetch), "err", err) } } @@ -361,7 +370,7 @@ func (s *stateObject) updateTrie() (Trie, error) { s.db.StorageDeleted.Add(1) } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.addrHash, s.data.Root, nil, used) + s.db.prefetcher.used(s.addrHash(), s.data.Root, nil, used) } s.uncommittedStorage = make(Storage) // empties the commit markers return tr, nil @@ -442,6 +451,12 @@ func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) { blob: s.code, } s.dirtyCode = false // reset the dirty flag + + if s.origin == nil { + op.code.originHash = types.EmptyCodeHash + } else { + op.code.originHash = common.BytesToHash(s.origin.CodeHash) + } } // Commit storage changes and the associated storage trie s.commitStorage(op) @@ -487,7 +502,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { obj := &stateObject{ db: db, address: s.address, - addrHash: s.addrHash, + addressHash: nil, origin: s.origin, data: s.data, code: s.code, @@ -501,11 +516,11 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { } switch s.trie.(type) { - case *trie.VerkleTrie: - // Verkle uses only one tree, and the copy has already been + case *bintrie.BinaryTrie: + // UBT uses only one tree, and the copy has already been // made in mustCopyTrie. obj.trie = db.trie - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: // Same thing for the transition tree, since the MPT is // read-only. obj.trie = db.trie @@ -534,6 +549,12 @@ func (s *stateObject) Code() []byte { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } + defer func(start time.Time) { + s.db.CodeLoaded += 1 + s.db.CodeReads += time.Since(start) + s.db.CodeLoadBytes += len(s.code) + }(time.Now()) + code, err := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) @@ -555,6 +576,11 @@ func (s *stateObject) CodeSize() int { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } + defer func(start time.Time) { + s.db.CodeLoaded += 1 + s.db.CodeReads += time.Since(start) + }(time.Now()) + size, err := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) diff --git a/core/state/state_sizer.go b/core/state/state_sizer.go index 2066c94845..02b73e5575 100644 --- a/core/state/state_sizer.go +++ b/core/state/state_sizer.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/triedb" "golang.org/x/sync/errgroup" ) @@ -48,6 +49,21 @@ var ( codeKeySize = int64(len(rawdb.CodePrefix) + common.HashLength) ) +// State size metrics +var ( + stateSizeChainHeightGauge = metrics.NewRegisteredGauge("state/height", nil) + stateSizeAccountsCountGauge = metrics.NewRegisteredGauge("state/accounts/count", nil) + stateSizeAccountsBytesGauge = metrics.NewRegisteredGauge("state/accounts/bytes", nil) + stateSizeStoragesCountGauge = metrics.NewRegisteredGauge("state/storages/count", nil) + stateSizeStoragesBytesGauge = metrics.NewRegisteredGauge("state/storages/bytes", nil) + stateSizeAccountTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/account/count", nil) + stateSizeAccountTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/account/bytes", nil) + stateSizeStorageTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/storage/count", nil) + stateSizeStorageTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/storage/bytes", nil) + stateSizeContractsCountGauge = metrics.NewRegisteredGauge("state/contracts/count", nil) + stateSizeContractsBytesGauge = metrics.NewRegisteredGauge("state/contracts/bytes", nil) +) + // SizeStats represents either the current state size statistics or the size // differences resulting from a state transition. type SizeStats struct { @@ -76,6 +92,20 @@ func (s SizeStats) String() string { ) } +func (s SizeStats) publish() { + stateSizeChainHeightGauge.Update(int64(s.BlockNumber)) + stateSizeAccountsCountGauge.Update(s.Accounts) + stateSizeAccountsBytesGauge.Update(s.AccountBytes) + stateSizeStoragesCountGauge.Update(s.Storages) + stateSizeStoragesBytesGauge.Update(s.StorageBytes) + stateSizeAccountTrieNodesCountGauge.Update(s.AccountTrienodes) + stateSizeAccountTrieNodesBytesGauge.Update(s.AccountTrienodeBytes) + stateSizeStorageTrieNodesCountGauge.Update(s.StorageTrienodes) + stateSizeStorageTrieNodesBytesGauge.Update(s.StorageTrienodeBytes) + stateSizeContractsCountGauge.Update(s.ContractCodes) + stateSizeContractsBytesGauge.Update(s.ContractCodeBytes) +} + // add applies the given state diffs and produces a new version of the statistics. func (s SizeStats) add(diff SizeStats) SizeStats { s.StateRoot = diff.StateRoot @@ -213,12 +243,14 @@ func calSizeStats(update *stateUpdate) (SizeStats, error) { } } - // Measure code changes. Note that the reported contract code size may be slightly - // inaccurate due to database deduplication (code is stored by its hash). However, - // this deviation is negligible and acceptable for measurement purposes. + codeExists := make(map[common.Hash]struct{}) for _, code := range update.codes { + if _, ok := codeExists[code.hash]; ok || code.duplicate { + continue + } stats.ContractCodes += 1 stats.ContractCodeBytes += codeKeySize + int64(len(code.blob)) + codeExists[code.hash] = struct{}{} } return stats, nil } @@ -309,8 +341,12 @@ func (t *SizeTracker) run() { stats[u.root] = stat last = u.root + // Publish statistics to metric system + stat.publish() + + // Evict the stale statistics heap.Push(&h, stats[u.root]) - for u.blockNumber-h[0].BlockNumber > statEvictThreshold { + for len(h) > 0 && u.blockNumber-h[0].BlockNumber > statEvictThreshold { delete(stats, h[0].StateRoot) heap.Pop(&h) } diff --git a/core/state/state_sizer_test.go b/core/state/state_sizer_test.go index 65f652e424..b3203afd74 100644 --- a/core/state/state_sizer_test.go +++ b/core/state/state_sizer_test.go @@ -58,7 +58,7 @@ func TestSizeTracker(t *testing.T) { state.AddBalance(addr3, uint256.NewInt(3000), tracing.BalanceChangeUnspecified) state.SetNonce(addr3, 3, tracing.NonceChangeUnspecified) - currentRoot, _, err := state.CommitWithUpdate(1, true, false) + currentRoot, err := state.Commit(1, true, false) if err != nil { t.Fatalf("Failed to commit initial state: %v", err) } @@ -83,7 +83,7 @@ func TestSizeTracker(t *testing.T) { if i%3 == 0 { newState.SetCode(testAddr, []byte{byte(i), 0x60, 0x80, byte(i + 1), 0x52}, tracing.CodeChangeUnspecified) } - root, _, err := newState.CommitWithUpdate(blockNum, true, false) + root, err := newState.Commit(blockNum, true, false) if err != nil { t.Fatalf("Failed to commit state at block %d: %v", blockNum, err) } @@ -154,21 +154,22 @@ func TestSizeTracker(t *testing.T) { if i%3 == 0 { newState.SetCode(testAddr, []byte{byte(i), 0x60, 0x80, byte(i + 1), 0x52}, tracing.CodeChangeUnspecified) } - root, update, err := newState.CommitWithUpdate(blockNum, true, false) + ret, err := newState.commitAndFlush(blockNum, true, false, true) if err != nil { t.Fatalf("Failed to commit state at block %d: %v", blockNum, err) } - if err := tdb.Commit(root, false); err != nil { + tracker.Notify(ret) + + if err := tdb.Commit(ret.root, false); err != nil { t.Fatalf("Failed to commit trie at block %d: %v", blockNum, err) } - diff, err := calSizeStats(update) + diff, err := calSizeStats(ret) if err != nil { t.Fatalf("Failed to calculate size stats for block %d: %v", blockNum, err) } trackedUpdates = append(trackedUpdates, diff) - tracker.Notify(update) - currentRoot = root + currentRoot = ret.root } finalRoot := rawdb.ReadSnapshotRoot(db) diff --git a/core/state/statedb.go b/core/state/statedb.go index cc005b451b..288a8e924c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -41,7 +41,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" "golang.org/x/sync/errgroup" ) @@ -146,15 +145,17 @@ type StateDB struct { witnessStats *stateless.WitnessStats // Measurements gathered during execution for debugging purposes - AccountReads time.Duration - AccountHashes time.Duration - AccountUpdates time.Duration - AccountCommits time.Duration + AccountReads time.Duration + AccountHashes time.Duration + AccountUpdates time.Duration + AccountCommits time.Duration + StorageReads time.Duration StorageUpdates time.Duration StorageCommits time.Duration SnapshotCommits time.Duration TrieDBCommits time.Duration + CodeReads time.Duration AccountLoaded int // Number of accounts retrieved from the database during the state transition AccountUpdated int // Number of accounts updated during the state transition @@ -163,6 +164,15 @@ type StateDB struct { StorageUpdated atomic.Int64 // Number of storage slots updated during the state transition StorageDeleted atomic.Int64 // Number of storage slots deleted during the state transition + // CodeLoadBytes is the total number of bytes read from contract code. + // This value may be smaller than the actual number of bytes read, since + // some APIs (e.g. CodeSize) may load the entire code from either the + // cache or the database when the size is not available in the cache. + CodeLoaded int // Number of contract code loaded during the state transition + CodeLoadBytes int // Total bytes of resolved code + CodeUpdated int // Number of contracts with code changes that persisted + CodeUpdateBytes int // Total bytes of persisted code written + deterministic bool recording bool } @@ -201,7 +211,7 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro transientStorage: newTransientStorage(), } if db.TrieDB().IsVerkle() { - sdb.accessEvents = NewAccessEvents(db.PointCache()) + sdb.accessEvents = NewAccessEvents() } return sdb, nil } @@ -310,6 +320,9 @@ func (s *StateDB) Logs() []*types.Log { for _, lgs := range s.logs { logs = append(logs, lgs...) } + sort.Slice(logs, func(i, j int) bool { + return logs[i].Index < logs[j].Index + }) return logs } @@ -571,22 +584,13 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common } // SelfDestruct marks the given account as selfdestructed. -// This clears the account balance. // // The account's state object is still available until the state is committed, // getStateObject will return a non-nil account after SelfDestruct. -func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int { +func (s *StateDB) SelfDestruct(addr common.Address) { stateObject := s.getStateObject(addr) - var prevBalance uint256.Int if stateObject == nil { - return prevBalance - } - prevBalance = *(stateObject.Balance()) - // Regardless of whether it is already destructed or not, we do have to - // journal the balance-change, if we set it to zero here. - if !stateObject.Balance().IsZero() { - s.arbExtraData.unexpectedBalanceDelta.Sub(s.arbExtraData.unexpectedBalanceDelta, stateObject.data.Balance.ToBig()) - stateObject.SetBalance(new(uint256.Int)) + return } // If it is already marked as self-destructed, we do not need to add it // for journalling a second time. @@ -594,18 +598,6 @@ func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int { s.journal.destruct(addr) stateObject.markSelfdestructed() } - return prevBalance -} - -func (s *StateDB) SelfDestruct6780(addr common.Address) (uint256.Int, bool) { - stateObject := s.getStateObject(addr) - if stateObject == nil { - return uint256.Int{}, false - } - if stateObject.newContract { - return s.SelfDestruct(addr), true - } - return *(stateObject.Balance()), false } // SetTransientState sets transient storage for a given account. It @@ -742,6 +734,16 @@ func (s *StateDB) CreateContract(addr common.Address) { } } +// IsNewContract reports whether the contract at the given address was deployed +// during the current transaction. +func (s *StateDB) IsNewContract(addr common.Address) bool { + obj := s.getStateObject(addr) + if obj == nil { + return false + } + return obj.newContract +} + // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. func (s *StateDB) Copy() *StateDB { @@ -993,13 +995,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { witness := trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } else if obj.trie != nil { witness := obj.trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } } @@ -1017,13 +1019,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { witness := trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } else if obj.trie != nil { witness := obj.trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } } @@ -1084,8 +1086,15 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if op.isDelete() { deletedAddrs = append(deletedAddrs, addr) } else { - s.updateStateObject(s.stateObjects[addr]) + obj := s.stateObjects[addr] + s.updateStateObject(obj) s.AccountUpdated += 1 + + // Count code writes post-Finalise so reverted CREATEs are excluded. + if obj.dirtyCode { + s.CodeUpdated += 1 + s.CodeUpdateBytes += len(obj.code) + } } usedAddrs = append(usedAddrs, addr) // Copy needed for closure } @@ -1435,7 +1444,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum return err } lock.Lock() - updates[obj.addrHash] = update + updates[obj.addrHash()] = update s.StorageCommits = time.Since(start) // overwrite with the longest storage commit runtime lock.Unlock() return nil @@ -1473,11 +1482,16 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum // commitAndFlush is a wrapper of commit which also commits the state mutations // to the configured data stores. -func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (*stateUpdate, error) { +func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool, deriveCodeFields bool) (*stateUpdate, error) { ret, err := s.commit(deleteEmptyObjects, noStorageWiping, block) if err != nil { return nil, err } + if deriveCodeFields { + if err := ret.deriveCodeFields(s.reader); err != nil { + return nil, err + } + } // Commit dirty contract code if any exists if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { batch := db.NewBatch() @@ -1546,17 +1560,17 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag // no empty accounts left that could be deleted by EIP-158, storage wiping // should not occur. func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (common.Hash, error) { - ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping) + ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping, false) if err != nil { return common.Hash{}, err } return ret.root, nil } -// CommitWithUpdate writes the state mutations and returns both the root hash and the state update. -// This is useful for tracking state changes at the blockchain level. +// CommitWithUpdate writes the state mutations and returns the state update for +// external processing (e.g., live tracing hooks or size tracker). func (s *StateDB) CommitWithUpdate(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (common.Hash, *stateUpdate, error) { - ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping) + ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping, true) if err != nil { return common.Hash{}, nil, err } @@ -1667,11 +1681,6 @@ func (s *StateDB) markUpdate(addr common.Address) { s.mutations[addr].typ = update } -// PointCache returns the point cache used by verkle tree. -func (s *StateDB) PointCache() *utils.PointCache { - return s.db.PointCache() -} - // Witness retrieves the current state witness being collected. func (s *StateDB) Witness() *stateless.Witness { return s.witness diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index b2963b9be8..9f35b8507f 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -228,7 +228,7 @@ func (test *stateTest) run() bool { } else { state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary } - ret, err := state.commitAndFlush(0, true, false) // call commit at the block boundary + ret, err := state.commitAndFlush(0, true, false, false) // call commit at the block boundary if err != nil { panic(err) } diff --git a/core/state/statedb_hooked.go b/core/state/statedb_hooked.go index c8b2b8204a..eccd104f79 100644 --- a/core/state/statedb_hooked.go +++ b/core/state/statedb_hooked.go @@ -17,7 +17,9 @@ package state import ( + "bytes" "math/big" + "sort" "github.com/ethereum/go-ethereum/arbitrum/filter" "github.com/ethereum/go-ethereum/common" @@ -27,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -55,6 +56,10 @@ func (s *hookedStateDB) CreateContract(addr common.Address) { s.inner.CreateContract(addr) } +func (s *hookedStateDB) IsNewContract(addr common.Address) bool { + return s.inner.IsNewContract(addr) +} + func (s *hookedStateDB) GetBalance(addr common.Address) *uint256.Int { return s.inner.GetBalance(addr) } @@ -135,10 +140,6 @@ func (s *hookedStateDB) AddSlotToAccessList(addr common.Address, slot common.Has s.inner.AddSlotToAccessList(addr, slot) } -func (s *hookedStateDB) PointCache() *utils.PointCache { - return s.inner.PointCache() -} - func (s *hookedStateDB) Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) { s.inner.Prepare(rules, sender, coinbase, dest, precompiles, txAccesses) } @@ -218,56 +219,8 @@ func (s *hookedStateDB) SetState(address common.Address, key common.Hash, value return prev } -func (s *hookedStateDB) SelfDestruct(address common.Address) uint256.Int { - var prevCode []byte - var prevCodeHash common.Hash - - if s.hooks.OnCodeChange != nil { - prevCode = s.inner.GetCode(address) - prevCodeHash = s.inner.GetCodeHash(address) - } - - prev := s.inner.SelfDestruct(address) - - if s.hooks.OnBalanceChange != nil && !prev.IsZero() { - s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct) - } - - if len(prevCode) > 0 { - if s.hooks.OnCodeChangeV2 != nil { - s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) - } else if s.hooks.OnCodeChange != nil { - s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil) - } - } - - return prev -} - -func (s *hookedStateDB) SelfDestruct6780(address common.Address) (uint256.Int, bool) { - var prevCode []byte - var prevCodeHash common.Hash - - if s.hooks.OnCodeChange != nil { - prevCodeHash = s.inner.GetCodeHash(address) - prevCode = s.inner.GetCode(address) - } - - prev, changed := s.inner.SelfDestruct6780(address) - - if s.hooks.OnBalanceChange != nil && !prev.IsZero() { - s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct) - } - - if changed && len(prevCode) > 0 { - if s.hooks.OnCodeChangeV2 != nil { - s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) - } else if s.hooks.OnCodeChange != nil { - s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil) - } - } - - return prev, changed +func (s *hookedStateDB) SelfDestruct(address common.Address) { + s.inner.SelfDestruct(address) } func (s *hookedStateDB) AddLog(log *types.Log) { @@ -279,19 +232,60 @@ func (s *hookedStateDB) AddLog(log *types.Log) { } func (s *hookedStateDB) Finalise(deleteEmptyObjects bool) { - defer s.inner.Finalise(deleteEmptyObjects) - if s.hooks.OnBalanceChange == nil { + if s.hooks.OnBalanceChange == nil && s.hooks.OnNonceChangeV2 == nil && s.hooks.OnNonceChange == nil && s.hooks.OnCodeChangeV2 == nil && s.hooks.OnCodeChange == nil { + // Short circuit if no relevant hooks are set. + s.inner.Finalise(deleteEmptyObjects) return } + + // Collect all self-destructed addresses first, then sort them to ensure + // that state change hooks will be invoked in deterministic + // order when the accounts are deleted below + var selfDestructedAddrs []common.Address for addr := range s.inner.journal.dirties { obj := s.inner.stateObjects[addr] - if obj != nil && obj.selfDestructed { - // If ether was sent to account post-selfdestruct it is burnt. + if obj == nil || !obj.selfDestructed { + // Not self-destructed, keep searching. + continue + } + selfDestructedAddrs = append(selfDestructedAddrs, addr) + } + sort.Slice(selfDestructedAddrs, func(i, j int) bool { + return bytes.Compare(selfDestructedAddrs[i][:], selfDestructedAddrs[j][:]) < 0 + }) + + for _, addr := range selfDestructedAddrs { + obj := s.inner.stateObjects[addr] + // Bingo: state object was self-destructed, call relevant hooks. + + // If ether was sent to account post-selfdestruct, record as burnt. + if s.hooks.OnBalanceChange != nil { if bal := obj.Balance(); bal.Sign() != 0 { s.hooks.OnBalanceChange(addr, bal.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestructBurn) } } + + // Nonce is set to reset on self-destruct. + if s.hooks.OnNonceChangeV2 != nil { + s.hooks.OnNonceChangeV2(addr, obj.Nonce(), 0, tracing.NonceChangeSelfdestruct) + } else if s.hooks.OnNonceChange != nil { + s.hooks.OnNonceChange(addr, obj.Nonce(), 0) + } + + // If an initcode invokes selfdestruct, do not emit a code change. + prevCodeHash := s.inner.GetCodeHash(addr) + if prevCodeHash == types.EmptyCodeHash { + continue + } + // Otherwise, trace the change. + if s.hooks.OnCodeChangeV2 != nil { + s.hooks.OnCodeChangeV2(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct) + } else if s.hooks.OnCodeChange != nil { + s.hooks.OnCodeChange(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil) + } } + + s.inner.Finalise(deleteEmptyObjects) } func (s *hookedStateDB) ActivateWasm(moduleHash common.Hash, asmMap map[rawdb.WasmTarget][]byte) error { diff --git a/core/state/statedb_hooked_test.go b/core/state/statedb_hooked_test.go index bacb7baee1..6fe17ec1b4 100644 --- a/core/state/statedb_hooked_test.go +++ b/core/state/statedb_hooked_test.go @@ -49,6 +49,8 @@ func TestBurn(t *testing.T) { createAndDestroy := func(addr common.Address) { hooked.AddBalance(addr, uint256.NewInt(100), tracing.BalanceChangeUnspecified) hooked.CreateContract(addr) + // Simulate what the opcode handler does: clear balance before selfdestruct + hooked.SubBalance(addr, hooked.GetBalance(addr), tracing.BalanceDecreaseSelfdestruct) hooked.SelfDestruct(addr) // sanity-check that balance is now 0 if have, want := hooked.GetBalance(addr), new(uint256.Int); !have.Eq(want) { @@ -122,9 +124,51 @@ func TestHooks(t *testing.T) { sdb.AddLog(&types.Log{ Address: common.Address{0xbb}, }) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + + for i, want := range wants { + if have := result[i]; have != want { + t.Fatalf("error event %d\nhave: %v\nwant: %v", i, have, want) + } + } +} + +func TestHooks_OnCodeChangeV2(t *testing.T) { + inner, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) + + var result []string + var wants = []string{ + "0xaa00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) ContractCreation", + "0xbb00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) ContractCreation", + "0xaa00000000000000000000000000000000000000.code: 0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + "0xbb00000000000000000000000000000000000000.code: 0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + } + emitF := func(format string, a ...any) { + result = append(result, fmt.Sprintf(format, a...)) + } + sdb := NewHookedState(inner, &tracing.Hooks{ + OnCodeChangeV2: func(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + emitF("%v.code: %#x (%v) ->%#x (%v) %s", addr, prevCode, prevCodeHash, code, codeHash, reason) + }, + }) + sdb.SetCode(common.Address{0xaa}, []byte{0x13, 37}, tracing.CodeChangeContractCreation) + sdb.SelfDestruct(common.Address{0xaa}) + + sdb.SetCode(common.Address{0xbb}, []byte{0x13, 38}, tracing.CodeChangeContractCreation) + sdb.CreateContract(common.Address{0xbb}) + sdb.SelfDestruct(common.Address{0xbb}) + sdb.Finalise(true) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + for i, want := range wants { if have := result[i]; have != want { - t.Fatalf("error event %d, have\n%v\nwant%v\n", i, have, want) + t.Fatalf("error event %d\nhave: %v\nwant: %v", i, have, want) } } } diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index f404613cd0..06bc479398 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -17,17 +17,27 @@ package state import ( + "fmt" "maps" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb" ) -// contractCode represents a contract code with associated metadata. +// contractCode represents contract bytecode along with its associated metadata. type contractCode struct { - hash common.Hash // hash is the cryptographic hash of the contract code. - blob []byte // blob is the binary representation of the contract code. + hash common.Hash // hash is the cryptographic hash of the current contract code. + blob []byte // blob is the binary representation of the current contract code. + originHash common.Hash // originHash is the cryptographic hash of the code before mutation. + + // Derived fields, populated only when state tracking is enabled. + duplicate bool // duplicate indicates whether the updated code already exists. + originBlob []byte // originBlob is the original binary representation of the contract code. } // accountDelete represents an operation for deleting an Ethereum account. @@ -82,8 +92,8 @@ type stateUpdate struct { storagesOrigin map[common.Address]map[common.Hash][]byte rawStorageKey bool - codes map[common.Address]contractCode // codes contains the set of dirty codes - nodes *trienode.MergedNodeSet // Aggregated dirty nodes caused by state changes + codes map[common.Address]*contractCode // codes contains the set of dirty codes + nodes *trienode.MergedNodeSet // Aggregated dirty nodes caused by state changes activatedWasms map[common.Hash]ActivatedWasm // newly activated WASMs } @@ -105,7 +115,7 @@ func newStateUpdate(rawStorageKey bool, originRoot common.Hash, root common.Hash accountsOrigin = make(map[common.Address][]byte) storages = make(map[common.Hash]map[common.Hash][]byte) storagesOrigin = make(map[common.Address]map[common.Hash][]byte) - codes = make(map[common.Address]contractCode) + codes = make(map[common.Address]*contractCode) ) // Since some accounts might be destroyed and recreated within the same // block, deletions must be aggregated first. @@ -127,7 +137,7 @@ func newStateUpdate(rawStorageKey bool, originRoot common.Hash, root common.Hash // Aggregate dirty contract codes if they are available. addr := op.address if op.code != nil { - codes[addr] = *op.code + codes[addr] = op.code } accounts[addrHash] = op.data @@ -193,3 +203,170 @@ func (sc *stateUpdate) stateSet() *triedb.StateSet { RawStorageKey: sc.rawStorageKey, } } + +// deriveCodeFields derives the missing fields of contract code changes +// such as original code value. +// +// Note: This operation is expensive and not needed during normal state +// transitions. It is only required when SizeTracker or StateUpdate hook +// is enabled to produce accurate state statistics. +func (sc *stateUpdate) deriveCodeFields(reader ContractCodeReader) error { + cache := make(map[common.Hash]bool) + for addr, code := range sc.codes { + if code.originHash != types.EmptyCodeHash { + blob, err := reader.Code(addr, code.originHash) + if err != nil { + return err + } + code.originBlob = blob + } + if exists, ok := cache[code.hash]; ok { + code.duplicate = exists + continue + } + res := reader.Has(addr, code.hash) + cache[code.hash] = res + code.duplicate = res + } + return nil +} + +// ToTracingUpdate converts the internal stateUpdate to an exported tracing.StateUpdate. +func (sc *stateUpdate) ToTracingUpdate() (*tracing.StateUpdate, error) { + update := &tracing.StateUpdate{ + OriginRoot: sc.originRoot, + Root: sc.root, + BlockNumber: sc.blockNumber, + AccountChanges: make(map[common.Address]*tracing.AccountChange, len(sc.accountsOrigin)), + StorageChanges: make(map[common.Address]map[common.Hash]*tracing.StorageChange), + CodeChanges: make(map[common.Address]*tracing.CodeChange, len(sc.codes)), + TrieChanges: make(map[common.Hash]map[string]*tracing.TrieNodeChange), + } + // Gather all account changes + for addr, oldData := range sc.accountsOrigin { + addrHash := crypto.Keccak256Hash(addr.Bytes()) + newData, exists := sc.accounts[addrHash] + if !exists { + return nil, fmt.Errorf("account %x not found", addr) + } + change := &tracing.AccountChange{} + + if len(oldData) > 0 { + acct, err := types.FullAccount(oldData) + if err != nil { + return nil, err + } + change.Prev = &types.StateAccount{ + Nonce: acct.Nonce, + Balance: acct.Balance, + Root: acct.Root, + CodeHash: acct.CodeHash, + } + } + if len(newData) > 0 { + acct, err := types.FullAccount(newData) + if err != nil { + return nil, err + } + change.New = &types.StateAccount{ + Nonce: acct.Nonce, + Balance: acct.Balance, + Root: acct.Root, + CodeHash: acct.CodeHash, + } + } + update.AccountChanges[addr] = change + } + + // Gather all storage slot changes + for addr, slots := range sc.storagesOrigin { + addrHash := crypto.Keccak256Hash(addr.Bytes()) + subset, exists := sc.storages[addrHash] + if !exists { + return nil, fmt.Errorf("storage %x not found", addr) + } + storageChanges := make(map[common.Hash]*tracing.StorageChange, len(slots)) + + for key, encPrev := range slots { + // Get new value - handle both raw and hashed key formats + var ( + exists bool + encNew []byte + decPrev []byte + decNew []byte + err error + ) + if sc.rawStorageKey { + encNew, exists = subset[crypto.Keccak256Hash(key.Bytes())] + } else { + encNew, exists = subset[key] + } + if !exists { + return nil, fmt.Errorf("storage slot %x-%x not found", addr, key) + } + + // Decode the prev and new values + if len(encPrev) > 0 { + _, decPrev, _, err = rlp.Split(encPrev) + if err != nil { + return nil, fmt.Errorf("failed to decode prevValue: %v", err) + } + } + if len(encNew) > 0 { + _, decNew, _, err = rlp.Split(encNew) + if err != nil { + return nil, fmt.Errorf("failed to decode newValue: %v", err) + } + } + storageChanges[key] = &tracing.StorageChange{ + Prev: common.BytesToHash(decPrev), + New: common.BytesToHash(decNew), + } + } + update.StorageChanges[addr] = storageChanges + } + + // Gather all contract code changes + for addr, code := range sc.codes { + change := &tracing.CodeChange{ + New: &tracing.ContractCode{ + Hash: code.hash, + Code: code.blob, + Exists: code.duplicate, + }, + } + if code.originHash != types.EmptyCodeHash { + change.Prev = &tracing.ContractCode{ + Hash: code.originHash, + Code: code.originBlob, + Exists: true, + } + } + update.CodeChanges[addr] = change + } + + // Gather all trie node changes + if sc.nodes != nil { + for owner, subset := range sc.nodes.Sets { + nodeChanges := make(map[string]*tracing.TrieNodeChange, len(subset.Origins)) + for path, oldNode := range subset.Origins { + newNode, exists := subset.Nodes[path] + if !exists { + return nil, fmt.Errorf("node %x-%v not found", owner, path) + } + nodeChanges[path] = &tracing.TrieNodeChange{ + Prev: &trienode.Node{ + Hash: crypto.Keccak256Hash(oldNode), + Blob: oldNode, + }, + New: &trienode.Node{ + Hash: newNode.Hash, + Blob: newNode.Blob, + }, + } + } + update.TrieChanges[owner] = nodeChanges + } + } + return update, nil +} diff --git a/core/state_processor.go b/core/state_processor.go index eace271136..8b432d8aae 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" @@ -26,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/params" ) @@ -56,7 +58,7 @@ func (p *StateProcessor) chainConfig() *params.ChainConfig { // Process returns the receipts and logs accumulated during the process and // returns the amount of gas that was used in the process. If any of the // transactions failed to execute due to insufficient gas it will return an error. -func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { +func (p *StateProcessor) Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { var ( config = p.chainConfig() receipts types.Receipts @@ -68,19 +70,20 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg gp = new(GasPool).AddGas(block.GasLimit()) ) + var tracingStateDB = vm.StateDB(statedb) + if hooks := cfg.Tracer; hooks != nil { + tracingStateDB = state.NewHookedState(statedb, hooks) + } + // Mutate the block and state according to any hard-fork specs if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(block.Number()) == 0 { - misc.ApplyDAOHardFork(statedb) + misc.ApplyDAOHardFork(tracingStateDB) } var ( context vm.BlockContext ) // Apply pre-execution system calls. - var tracingStateDB = vm.StateDB(statedb) - if hooks := cfg.Tracer; hooks != nil { - tracingStateDB = state.NewHookedState(statedb, hooks) - } context = NewEVMBlockContext(header, p.chain, nil) evm := vm.NewEVM(context, tracingStateDB, config, cfg) @@ -103,43 +106,58 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.SetTxContext(tx.Hash(), i) - + _, _, spanEnd := telemetry.StartSpan(ctx, "core.ApplyTransactionWithEVM", + telemetry.StringAttribute("tx.hash", tx.Hash().Hex()), + telemetry.Int64Attribute("tx.index", int64(i)), + ) receipt, _, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, usedGas, evm, nil) + spanEnd(&err) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) } + requests, err := postExecution(ctx, config, block, allLogs, evm) + if err != nil { + return nil, err + } + + // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) + p.chain.Engine().Finalize(p.chain, header, tracingStateDB, block.Body()) + + return &ProcessResult{ + Receipts: receipts, + Requests: requests, + Logs: allLogs, + GasUsed: *usedGas, + }, nil +} + +// postExecution processes the post-execution system calls if Prague is enabled. +func postExecution(ctx context.Context, config *params.ChainConfig, block *types.Block, allLogs []*types.Log, evm *vm.EVM) (requests [][]byte, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.postExecution") + defer spanEnd(&err) // Read requests if Prague is enabled. - var requests [][]byte // Arbitrum has no Deposit, Witdrawal, or Consolidation requests. - if !config.IsArbitrum() && config.IsPrague(block.Number(), block.Time(), context.ArbOSVersion) { + if !config.IsArbitrum() && config.IsPrague(block.Number(), block.Time(), evm.Context.ArbOSVersion) { requests = [][]byte{} // EIP-6110 if err := ParseDepositLogs(&requests, allLogs, config); err != nil { - return nil, fmt.Errorf("failed to parse deposit logs: %w", err) + return requests, fmt.Errorf("failed to parse deposit logs: %w", err) } // EIP-7002 if err := ProcessWithdrawalQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process withdrawal queue: %w", err) + return requests, fmt.Errorf("failed to process withdrawal queue: %w", err) } // EIP-7251 if err := ProcessConsolidationQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process consolidation queue: %w", err) + return requests, fmt.Errorf("failed to process consolidation queue: %w", err) } } - // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) - p.chain.Engine().Finalize(p.chain, header, tracingStateDB, block.Body()) - - return &ProcessResult{ - Receipts: receipts, - Requests: requests, - Logs: allLogs, - GasUsed: *usedGas, - }, nil + return requests, nil } // ApplyTransactionWithEVM attempts to apply a transaction to the given state database diff --git a/core/state_transition_test.go b/core/state_transition_test.go index 32fc0322e7..1993aec6df 100644 --- a/core/state_transition_test.go +++ b/core/state_transition_test.go @@ -2,7 +2,7 @@ package core import ( "bytes" - "errors" + "fmt" "testing" "github.com/ethereum/go-ethereum/arbitrum/multigas" @@ -339,12 +339,8 @@ func TestCallVariantsMultiGas(t *testing.T) { callFn: func() ([]byte, uint64, multigas.MultiGas, error) { return evm.StaticCall(caller, contractAddr, nil, gasLimit) }, - expectErr: vm.ErrWriteProtection, - expectedMultiGas: multigas.MultiGasFromPairs( - multigas.Pair{Kind: multigas.ResourceKindComputation, Amount: refinedComputationGas}, - multigas.Pair{Kind: multigas.ResourceKindStorageAccessRead, Amount: params.ColdSloadCostEIP2929}, - multigas.Pair{Kind: multigas.ResourceKindStorageGrowth, Amount: params.SstoreSetGasEIP2200}, - ), + expectErr: fmt.Errorf("%w: %v", vm.ErrOutOfGas, vm.ErrWriteProtection), + expectedMultiGas: multigas.ComputationGas(refinedComputationGas + params.ColdSloadCostEIP2929 + params.SstoreSetGasEIP2200), }, } @@ -352,8 +348,15 @@ func TestCallVariantsMultiGas(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ret, leftover, usedMultiGas, err := tt.callFn() - if !errors.Is(err, tt.expectErr) { - t.Fatalf("unexpected error: got %v, want %v", err, tt.expectErr) + if tt.expectErr != nil { + if err == nil { + t.Fatalf("unexpected error, got: nil, want: %v", tt.expectErr) + } + if err.Error() != tt.expectErr.Error() { + t.Fatalf("unexpected error, got: %v, want: %v", err, tt.expectErr) + } + } else if err != nil { + t.Fatalf("unexpected error, got: %v, want: nil", err) } if gasLimit-leftover != usedMultiGas.SingleGas() { diff --git a/core/stateless.go b/core/stateless.go index b20c909da6..88d8ed8138 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -17,6 +17,8 @@ package core import ( + "context" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus/beacon" @@ -40,7 +42,7 @@ import ( // - It cannot be placed outside of core, because it needs to construct a dud headerchain // // TODO(karalabe): Would be nice to resolve both issues above somehow and move it. -func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { +func ExecuteStateless(ctx context.Context, config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { // Sanity check if the supplied block accidentally contains a set root or // receipt hash. If so, be very loud, but still continue. if block.Root() != (common.Hash{}) { @@ -66,7 +68,7 @@ func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *typ validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block // Run the stateless blocks processing and self-validate certain fields - res, err := processor.Process(block, db, vmconfig) + res, err := processor.Process(ctx, block, db, vmconfig) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 94f5587f99..73ce031bff 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -62,10 +62,17 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { // If current path is a prefix of the next path, it's not a leaf. // The last path is always a leaf. if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { + depth := len(path) if owner == (common.Hash{}) { - s.accountTrieLeaves[len(path)] += 1 + if depth >= len(s.accountTrieLeaves) { + depth = len(s.accountTrieLeaves) - 1 + } + s.accountTrieLeaves[depth] += 1 } else { - s.storageTrieLeaves[len(path)] += 1 + if depth >= len(s.storageTrieLeaves) { + depth = len(s.storageTrieLeaves) - 1 + } + s.storageTrieLeaves[depth] += 1 } } } diff --git a/core/tracing/gen_code_change_reason_stringer.go b/core/tracing/gen_code_change_reason_stringer.go index 9372954063..2531b10471 100644 --- a/core/tracing/gen_code_change_reason_stringer.go +++ b/core/tracing/gen_code_change_reason_stringer.go @@ -22,8 +22,9 @@ const _CodeChangeReason_name = "UnspecifiedContractCreationGenesisAuthorizationA var _CodeChangeReason_index = [...]uint8{0, 11, 27, 34, 47, 65, 77, 83} func (i CodeChangeReason) String() string { - if i >= CodeChangeReason(len(_CodeChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeChangeReason_index)-1 { return "CodeChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeChangeReason_name[_CodeChangeReason_index[i]:_CodeChangeReason_index[i+1]] + return _CodeChangeReason_name[_CodeChangeReason_index[idx]:_CodeChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_nonce_change_reason_stringer.go b/core/tracing/gen_nonce_change_reason_stringer.go index f775c1f3a6..8c9099b7ce 100644 --- a/core/tracing/gen_nonce_change_reason_stringer.go +++ b/core/tracing/gen_nonce_change_reason_stringer.go @@ -15,15 +15,17 @@ func _() { _ = x[NonceChangeNewContract-4] _ = x[NonceChangeAuthorization-5] _ = x[NonceChangeRevert-6] + _ = x[NonceChangeSelfdestruct-7] } -const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevert" +const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevertSelfdestruct" -var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70} +var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70, 82} func (i NonceChangeReason) String() string { - if i >= NonceChangeReason(len(_NonceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_NonceChangeReason_index)-1 { return "NonceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _NonceChangeReason_name[_NonceChangeReason_index[i]:_NonceChangeReason_index[i+1]] + return _NonceChangeReason_name[_NonceChangeReason_index[idx]:_NonceChangeReason_index[idx+1]] } diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 24aeae76d0..1cdb2f1059 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" ) @@ -80,6 +81,56 @@ type BlockEvent struct { Safe *types.Header } +// StateUpdate represents the state mutations resulting from block execution. +// It provides access to account changes, storage changes, and contract code +// deployments with both previous and new values. +type StateUpdate struct { + OriginRoot common.Hash // State root before the update + Root common.Hash // State root after the update + BlockNumber uint64 + + // AccountChanges contains all account state changes keyed by address. + AccountChanges map[common.Address]*AccountChange + + // StorageChanges contains all storage slot changes keyed by address and storage slot key. + StorageChanges map[common.Address]map[common.Hash]*StorageChange + + // CodeChanges contains all contract code changes keyed by address. + CodeChanges map[common.Address]*CodeChange + + // TrieChanges contains trie node mutations keyed by address hash and trie node path. + TrieChanges map[common.Hash]map[string]*TrieNodeChange +} + +// AccountChange represents a change to an account's state. +type AccountChange struct { + Prev *types.StateAccount // nil if account was created + New *types.StateAccount // nil if account was deleted +} + +// StorageChange represents a change to a storage slot. +type StorageChange struct { + Prev common.Hash // previous value (zero if slot was created) + New common.Hash // new value (zero if slot was deleted) +} + +type ContractCode struct { + Hash common.Hash + Code []byte + Exists bool // true if the code was existent +} + +// CodeChange represents a change in contract code of an account. +type CodeChange struct { + Prev *ContractCode // nil if no code existed before + New *ContractCode +} + +type TrieNodeChange struct { + Prev *trienode.Node + New *trienode.Node +} + type ( /* - VM events - @@ -169,6 +220,11 @@ type ( // beacon block root. OnSystemCallEndHook = func() + // StateUpdateHook is called after state is committed for a block. + // It provides access to the complete state mutations including account changes, + // storage changes, trie node mutations, and contract code deployments. + StateUpdateHook = func(update *StateUpdate) + /* - State events - */ @@ -224,6 +280,7 @@ type Hooks struct { OnSystemCallStart OnSystemCallStartHook OnSystemCallStartV2 OnSystemCallStartHookV2 OnSystemCallEnd OnSystemCallEndHook + OnStateUpdate StateUpdateHook // State events OnBalanceChange BalanceChangeHook OnNonceChange NonceChangeHook @@ -466,6 +523,9 @@ const ( // NonceChangeRevert is emitted when the nonce is reverted back to a previous value due to call failure. // It is only emitted when the tracer has opted in to use the journaling wrapper (WrapWithJournal). NonceChangeRevert NonceChangeReason = 6 + + // NonceChangeSelfdestruct is emitted when the nonce is reset to zero due to a self-destruct + NonceChangeSelfdestruct NonceChangeReason = 7 ) // CodeChangeReason is used to indicate the reason for a code change. diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 11a1b8b866..468e1bb42f 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -21,12 +21,10 @@ import ( "container/heap" "errors" "fmt" - "maps" "math" "math/big" "os" "path/filepath" - "slices" "sort" "sync" "sync/atomic" @@ -97,10 +95,15 @@ const ( // store. storeVersion = 1 - // conversionTimeWindow defines the period after the Osaka fork during which - // the pool will still accept and convert legacy blob transactions. After this - // window, all legacy blob transactions will be rejected. - conversionTimeWindow = time.Hour * 2 + // gappedLifetime is the approximate duration for which nonce-gapped transactions + // are kept before being dropped. Since gapped is only a reorder buffer and it + // is expected that the original transactions were inserted in the mempool in + // nonce order, the duration is kept short to avoid DoS vectors. + gappedLifetime = 1 * time.Minute + + // maxGappedTxs is the maximum number of gapped transactions kept overall. + // This is a safety limit to avoid DoS vectors. + maxGapped = 128 ) // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and @@ -337,9 +340,11 @@ type BlobPool struct { stored uint64 // Useful data size of all transactions on disk limbo *limbo // Persistent data store for the non-finalized blobs - signer types.Signer // Transaction signer to use for sender recovery - chain BlockChain // Chain object to access the state through - cQueue *conversionQueue // The queue for performing legacy sidecar conversion (TODO: remove after Osaka) + gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high) + gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion + + signer types.Signer // Transaction signer to use for sender recovery + chain BlockChain // Chain object to access the state through head atomic.Pointer[types.Header] // Current head of the chain state *state.StateDB // Current state at the head of the chain @@ -368,16 +373,22 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo hasPendingAuth: hasPendingAuth, signer: types.LatestSigner(chain.Config()), chain: chain, - cQueue: newConversionQueue(), // Deprecate it after the osaka fork lookup: newLookup(), index: make(map[common.Address][]*blobTxMeta), spent: make(map[common.Address]*uint256.Int), + gapped: make(map[common.Address][]*types.Transaction), + gappedSource: make(map[common.Hash]common.Address), } } // Filter returns whether the given transaction can be consumed by the blob pool. func (p *BlobPool) Filter(tx *types.Transaction) bool { - return tx.Type() == types.BlobTxType + return p.FilterType(tx.Type()) +} + +// FilterType returns whether the blob pool supports the given transaction type. +func (p *BlobPool) FilterType(kind byte) bool { + return kind == types.BlobTxType } // Init sets the gas price needed to keep a transaction in the pool and the chain @@ -414,7 +425,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser p.state = state // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(p.chain.Config())) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the queue blob store after fusaka slotter, err = tryMigrate(p.chain.Config(), slotter, queuedir) @@ -485,9 +496,6 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser // Close closes down the underlying persistent store. func (p *BlobPool) Close() error { - // Terminate the conversion queue - p.cQueue.close() - var errs []error if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set if err := p.limbo.Close(); err != nil { @@ -841,6 +849,9 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { resettimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + // Handle reorg buffer timeouts evicting old gapped transactions + p.evictGapped() + statedb, err := p.chain.StateAt(newHead.Root) if err != nil { log.Error("Failed to reset blobpool state", "err", err) @@ -887,172 +898,6 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { basefeeGauge.Update(int64(basefee.Uint64())) blobfeeGauge.Update(int64(blobfee.Uint64())) p.updateStorageMetrics() - - // Perform the conversion logic at the fork boundary - if !p.chain.Config().IsOsaka(oldHead.Number, oldHead.Time, types.DeserializeHeaderExtraInformation(oldHead).ArbOSFormatVersion) && p.chain.Config().IsOsaka(newHead.Number, newHead.Time, types.DeserializeHeaderExtraInformation(newHead).ArbOSFormatVersion) { - // Deep copy all indexed transaction metadata. - var ( - ids = make(map[common.Address]map[uint64]uint64) - txs = make(map[common.Address]map[uint64]common.Hash) - ) - for sender, list := range p.index { - ids[sender] = make(map[uint64]uint64) - txs[sender] = make(map[uint64]common.Hash) - for _, m := range list { - ids[sender][m.nonce] = m.id - txs[sender][m.nonce] = m.hash - } - } - // Initiate the background conversion thread. - p.cQueue.launchBillyConversion(func() { - p.convertLegacySidecars(ids, txs) - }) - } -} - -// compareAndSwap checks if the specified transaction is still tracked in the pool -// and replace the metadata accordingly. It should only be used in the fork boundary -// bulk conversion. If it fails for some reason, the subsequent txs won't be dropped -// for simplicity which we assume it's very likely to happen. -// -// The returned flag indicates whether the replacement succeeded. -func (p *BlobPool) compareAndSwap(address common.Address, hash common.Hash, blob []byte, oldID uint64, oldStorageSize uint32) bool { - p.lock.Lock() - defer p.lock.Unlock() - - newId, err := p.store.Put(blob) - if err != nil { - log.Error("Failed to store transaction", "hash", hash, "err", err) - return false - } - newSize := uint64(len(blob)) - newStorageSize := p.store.Size(newId) - - // Terminate the procedure if the transaction was already evicted. The - // newly added blob should be removed before return. - if !p.lookup.update(hash, newId, newSize) { - if derr := p.store.Delete(newId); derr != nil { - log.Error("Failed to delete the dangling blob tx", "err", derr) - } else { - log.Warn("Deleted the dangling blob tx", "id", newId) - } - return false - } - // Update the metadata of blob transaction - for _, meta := range p.index[address] { - if meta.hash == hash { - meta.id = newId - meta.version = types.BlobSidecarVersion1 - meta.storageSize = newStorageSize - meta.size = newSize - - p.stored += uint64(newStorageSize) - p.stored -= uint64(oldStorageSize) - break - } - } - if err := p.store.Delete(oldID); err != nil { - log.Error("Failed to delete the legacy transaction", "hash", hash, "id", oldID, "err", err) - } - return true -} - -// convertLegacySidecar fetches transaction data from the store, performs an -// on-the-fly conversion. This function is intended for use only during the -// Osaka fork transition period. -// -// The returned flag indicates whether the replacement succeeds or not. -func (p *BlobPool) convertLegacySidecar(sender common.Address, hash common.Hash, id uint64) bool { - start := time.Now() - - // Retrieves the legacy blob transaction from the underlying store with - // read lock held, preventing any potential data race around the slot - // specified by the id. - p.lock.RLock() - data, err := p.store.Get(id) - if err != nil { - p.lock.RUnlock() - // The transaction may have been evicted simultaneously, safe to skip conversion. - log.Debug("Blob transaction is missing", "hash", hash, "id", id, "err", err) - return false - } - oldStorageSize := p.store.Size(id) - p.lock.RUnlock() - - // Decode the transaction, the failure is not expected and report the error - // loudly if possible. If the blob transaction in this slot is corrupted, - // leave it in the store, it will be dropped during the next pool - // initialization. - var tx types.Transaction - if err = rlp.DecodeBytes(data, &tx); err != nil { - log.Error("Blob transaction is corrupted", "hash", hash, "id", id, "err", err) - return false - } - - // Skip conversion if the transaction does not match the expected hash, or if it was - // already converted. This can occur if the original transaction was evicted from the - // pool and the slot was reused by a new one. - if tx.Hash() != hash { - log.Warn("Blob transaction was replaced", "hash", hash, "id", id, "stored", tx.Hash()) - return false - } - sc := tx.BlobTxSidecar() - if sc.Version >= types.BlobSidecarVersion1 { - log.Debug("Skipping conversion of blob tx", "hash", hash, "id", id) - return false - } - - // Perform the sidecar conversion, the failure is not expected and report the error - // loudly if possible. - if err := tx.BlobTxSidecar().ToV1(); err != nil { - log.Error("Failed to convert blob transaction", "hash", hash, "err", err) - return false - } - - // Encode the converted transaction, the failure is not expected and report - // the error loudly if possible. - blob, err := rlp.EncodeToBytes(&tx) - if err != nil { - log.Error("Failed to encode blob transaction", "hash", tx.Hash(), "err", err) - return false - } - - // Replace the legacy blob transaction with the converted format. - if !p.compareAndSwap(sender, hash, blob, id, oldStorageSize) { - log.Error("Failed to replace the legacy transaction", "hash", hash) - return false - } - log.Debug("Converted legacy blob transaction", "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) - return true -} - -// convertLegacySidecars converts all given transactions to sidecar version 1. -// -// If any of them fails to be converted, the subsequent transactions will still -// be processed, as we assume the failure is very unlikely to happen. If happens, -// these transactions will be stuck in the pool until eviction. -func (p *BlobPool) convertLegacySidecars(ids map[common.Address]map[uint64]uint64, txs map[common.Address]map[uint64]common.Hash) { - var ( - start = time.Now() - success int - failure int - ) - for addr, list := range txs { - // Transactions evicted from the pool must be contiguous, if in any case, - // the transactions are gapped with each other, they will be discarded. - nonces := slices.Collect(maps.Keys(list)) - slices.Sort(nonces) - - // Convert the txs with nonce order - for _, nonce := range nonces { - if p.convertLegacySidecar(addr, list[nonce], ids[addr][nonce]) { - success++ - } else { - failure++ - } - } - } - log.Info("Completed blob transaction conversion", "discarded", failure, "injected", success, "elapsed", common.PrettyDuration(time.Since(start))) } // reorg assembles all the transactors and missing transactions between an old @@ -1371,7 +1216,9 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { State: p.state, FirstNonceGap: func(addr common.Address) uint64 { - // Nonce gaps are not permitted in the blob pool, the first gap will + // Nonce gaps are permitted in the blob pool, but only as part of the + // in-memory 'gapped' buffer. We expose the gap here to validateTx, + // then handle the error by adding to the buffer. The first gap will // be the next nonce shifted by however many transactions we already // have pooled. return p.state.GetNonce(addr) + uint64(len(p.index[addr])) @@ -1450,7 +1297,9 @@ func (p *BlobPool) Has(hash common.Hash) bool { p.lock.RLock() defer p.lock.RUnlock() - return p.lookup.exists(hash) + poolHas := p.lookup.exists(hash) + _, gapped := p.gappedSource[hash] + return poolHas || gapped } func (p *BlobPool) getRLP(hash common.Hash) []byte { @@ -1532,8 +1381,8 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { // // The version argument specifies the type of proofs to return, either the // blob proofs (version 0) or the cell proofs (version 1). Proofs conversion is -// CPU intensive, so only done if explicitly requested with the convert flag. -func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { +// CPU intensive and prohibited in the blobpool explicitly. +func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { var ( blobs = make([]*kzg4844.Blob, len(vhashes)) commitments = make([]kzg4844.Commitment, len(vhashes)) @@ -1584,7 +1433,7 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( } // Mark hash as seen. filled[hash] = struct{}{} - if sidecar.Version != version && !convert { + if sidecar.Version != version { // Skip blobs with incompatible version. Note we still track the blob hash // in `filled` here, ensuring that we do not resolve this tx another time. continue @@ -1593,29 +1442,13 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( var pf []kzg4844.Proof switch version { case types.BlobSidecarVersion0: - if sidecar.Version == types.BlobSidecarVersion0 { - pf = []kzg4844.Proof{sidecar.Proofs[i]} - } else { - proof, err := kzg4844.ComputeBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i]) - if err != nil { - return nil, nil, nil, err - } - pf = []kzg4844.Proof{proof} - } + pf = []kzg4844.Proof{sidecar.Proofs[i]} case types.BlobSidecarVersion1: - if sidecar.Version == types.BlobSidecarVersion0 { - cellProofs, err := kzg4844.ComputeCellProofs(&sidecar.Blobs[i]) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs - } else { - cellProofs, err := sidecar.CellProofsAt(i) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs + cellProofs, err := sidecar.CellProofsAt(i) + if err != nil { + return nil, nil, nil, err } + pf = cellProofs } for _, index := range list { blobs[index] = &sidecar.Blobs[i] @@ -1642,66 +1475,21 @@ func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int { return available } -// preCheck performs the static validation upon the provided tx and converts -// the legacy sidecars if Osaka fork has been activated with a short time window. -// -// This function is pure static and lock free. -func (p *BlobPool) preCheck(tx *types.Transaction) error { - var ( - head = p.head.Load() - isOsaka = p.chain.Config().IsOsaka(head.Number, head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) - deadline time.Time - ) - if isOsaka { - deadline = time.Unix(int64(*p.chain.Config().OsakaTime), 0).Add(conversionTimeWindow) - } - // Validate the transaction statically at first to avoid unnecessary - // conversion. This step doesn't require lock protection. - if err := p.ValidateTxBasics(tx); err != nil { - return err - } - // Before the Osaka fork, reject the blob txs with cell proofs - if !isOsaka { - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion0 { - return nil - } else { - return errors.New("cell proof is not supported yet") - } - } - // After the Osaka fork, reject the legacy blob txs if the conversion - // time window is passed. - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion1 { - return nil - } - if head.Time > uint64(deadline.Unix()) { - return errors.New("legacy blob tx is not supported") - } - // Convert the legacy sidecar after Osaka fork. This could be a long - // procedure which takes a few seconds, even minutes if there is a long - // queue. Fortunately it will only block the routine of the source peer - // announcing the tx, without affecting other parts. - return p.cQueue.convert(tx) -} - // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { var ( - errs []error = make([]error, len(txs)) - adds = make([]*types.Transaction, 0, len(txs)) + errs = make([]error, len(txs)) + adds = make([]*types.Transaction, 0, len(txs)) ) for i, tx := range txs { - if errs[i] = p.preCheck(tx); errs[i] != nil { + if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil { continue } if errs[i] = p.add(tx); errs[i] == nil { adds = append(adds, tx.WithoutBlobTxSidecar()) } } - if len(adds) > 0 { - p.discoverFeed.Send(core.NewTxsEvent{Txs: adds}) - p.insertFeed.Send(core.NewTxsEvent{Txs: adds}) - } return errs } @@ -1720,6 +1508,13 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addtimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + return p.addLocked(tx, true) +} + +// addLocked inserts a new blob transaction into the pool if it passes validation (both +// consensus validity and pool restrictions). It must be called with the pool lock held. +// Only for internal use. +func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) @@ -1732,6 +1527,21 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addStaleMeter.Mark(1) case errors.Is(err, core.ErrNonceTooHigh): addGappedMeter.Mark(1) + // Store the tx in memory, and revalidate later + from, _ := types.Sender(p.signer, tx) + allowance := p.gappedAllowance(from) + if allowance >= 1 && len(p.gapped) < maxGapped { + p.gapped[from] = append(p.gapped[from], tx) + p.gappedSource[tx.Hash()] = from + log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + return nil + } else { + // if maxGapped is reached, it is better to give time to gapped + // transactions by keeping the old and dropping this one. + // Thus replacing a gapped transaction with another gapped transaction + // is discouraged. + log.Trace("no gapped blob queue allowance", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } case errors.Is(err, core.ErrInsufficientFunds): addOverdraftedMeter.Mark(1) case errors.Is(err, txpool.ErrAccountLimitExceeded): @@ -1869,6 +1679,58 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { p.updateStorageMetrics() addValidMeter.Mark(1) + + // Notify all listeners of the new arrival + p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + + //check the gapped queue for this account and try to promote + if gtxs, ok := p.gapped[from]; checkGapped && ok && len(gtxs) > 0 { + // We have to add in nonce order, but we want to stable sort to cater for situations + // where transactions are replaced, keeping the original receive order for same nonce + sort.SliceStable(gtxs, func(i, j int) bool { + return gtxs[i].Nonce() < gtxs[j].Nonce() + }) + for len(gtxs) > 0 { + stateNonce := p.state.GetNonce(from) + firstgap := stateNonce + uint64(len(p.index[from])) + + if gtxs[0].Nonce() > firstgap { + // Anything beyond the first gap is not addable yet + break + } + + // Drop any buffered transactions that became stale in the meantime (included in chain or replaced) + // If we arrive to the transaction in the pending range (between the state Nonce and first gap, we + // try to add them now while removing from here. + tx := gtxs[0] + gtxs[0] = nil + gtxs = gtxs[1:] + delete(p.gappedSource, tx.Hash()) + + if tx.Nonce() < stateNonce { + // Stale, drop it. Eventually we could add to limbo here if hash matches. + log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from])) + continue + } + + if tx.Nonce() <= firstgap { + // If we hit the pending range, including the first gap, add it and continue to try to add more. + // We do not recurse here, but continue to loop instead. + // We are under lock, so we can add the transaction directly. + if err := p.addLocked(tx, false); err == nil { + log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } else { + log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err) + } + } + } + if len(gtxs) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = gtxs + } + } return nil } @@ -2100,6 +1962,50 @@ func (p *BlobPool) Nonce(addr common.Address) uint64 { return p.state.GetNonce(addr) } +// gappedAllowance returns the number of gapped transactions still +// allowed for the given account. Allowance is based on a slow-start +// logic, allowing more gaps (resource usage) to accounts with a +// higher nonce. Can also return negative values. +func (p *BlobPool) gappedAllowance(addr common.Address) int { + // Gaps happen, but we don't want to allow too many. + // Use log10(nonce+1) as the allowance, with a minimum of 0. + nonce := p.state.GetNonce(addr) + allowance := int(math.Log10(float64(nonce + 1))) + // Cap the allowance to the remaining pool space + return min(allowance, maxTxsPerAccount-len(p.index[addr])) - len(p.gapped[addr]) +} + +// evictGapped removes the old transactions from the gapped reorder buffer. +// Concurrency: The caller must hold the pool lock before calling this function. +func (p *BlobPool) evictGapped() { + cutoff := time.Now().Add(-gappedLifetime) + for from, txs := range p.gapped { + nonce := p.state.GetNonce(from) + // Reuse the original slice to avoid extra allocations. + // This is safe because we only keep references to the original gappedTx objects, + // and we overwrite the slice for this account after filtering. + keep := txs[:0] + for i, gtx := range txs { + if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce { + // Evict old or stale transactions + // Should we add stale to limbo here if it would belong? + delete(p.gappedSource, gtx.Hash()) + txs[i] = nil // Explicitly nil out evicted element + } else { + keep = append(keep, gtx) + } + } + if len(keep) < len(txs) { + log.Trace("Evicting old gapped blob transactions", "count", len(txs)-len(keep), "from", from) + } + if len(keep) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = keep + } + } +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *BlobPool) Stats() (int, int) { @@ -2134,9 +2040,15 @@ func (p *BlobPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*ty // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { - if p.Has(hash) { + p.lock.RLock() + defer p.lock.RUnlock() + + if p.lookup.exists(hash) { return txpool.TxStatusPending } + if _, gapped := p.gappedSource[hash]; gapped { + return txpool.TxStatusQueued + } return txpool.TxStatusUnknown } @@ -2186,6 +2098,11 @@ func (p *BlobPool) Clear() { p.index = make(map[common.Address][]*blobTxMeta) p.spent = make(map[common.Address]*uint256.Int) + // Reset counters and the gapped buffer + p.stored = 0 + p.gapped = make(map[common.Address][]*types.Transaction) + p.gappedSource = make(map[common.Hash]common.Address) + var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head.Load())) blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index f7d8ca209b..4bb3567b69 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -92,10 +92,6 @@ type testBlockChain struct { blockTime *uint64 } -func (bc *testBlockChain) setHeadTime(time uint64) { - bc.blockTime = &time -} - func (bc *testBlockChain) Config() *params.ChainConfig { return bc.config } @@ -433,11 +429,11 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { hashes = append(hashes, tx.vhashes...) } } - blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { t.Fatal(err) } - blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1, false) + blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { t.Fatal(err) } @@ -1329,7 +1325,7 @@ func TestBlobCountLimit(t *testing.T) { // Check that first succeeds second fails. if errs[0] != nil { - t.Fatalf("expected tx with 7 blobs to succeed") + t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0]) } if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) { t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1]) @@ -1356,9 +1352,10 @@ func TestAdd(t *testing.T) { } // addtx is a helper sender/tx tuple to represent a new tx addition type addtx struct { - from string - tx *types.BlobTx - err error + from string + tx *types.BlobTx + err error + check func(*BlobPool, *types.Transaction) bool } tests := []struct { @@ -1375,6 +1372,7 @@ func TestAdd(t *testing.T) { "bob": {balance: 21100 + blobSize, nonce: 1}, "claire": {balance: 21100 + blobSize}, "dave": {balance: 21100 + blobSize, nonce: 1}, + "eve": {balance: 21100 + blobSize, nonce: 10}, // High nonce to test gapped acceptance }, adds: []addtx{ { // New account, no previous txs: accept nonce 0 @@ -1402,6 +1400,14 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(2, 1, 1, 1), err: core.ErrNonceTooHigh, }, + { // Old account, 10 txs in chain: 0 pending: accept nonce 11 as gapped + from: "eve", + tx: makeUnsignedTx(11, 1, 1, 1), + err: nil, + check: func(pool *BlobPool, tx *types.Transaction) bool { + return pool.Status(tx.Hash()) == txpool.TxStatusQueued + }, + }, }, }, // Transactions from already pooled accounts should only be accepted if @@ -1762,15 +1768,28 @@ func TestAdd(t *testing.T) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, errs[0], add.err) } if add.err == nil { - size, exist := pool.lookup.sizeOfTx(signed.Hash()) - if !exist { - t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + // first check if tx is in the pool (reorder queue or pending) + if !pool.Has(signed.Hash()) { + t.Errorf("test %d, tx %d: added transaction not found in pool", i, j) } - if size != signed.Size() { - t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", - i, j, size, signed.Size()) + // if it is pending, check if size matches + if pool.Status(signed.Hash()) == txpool.TxStatusPending { + size, exist := pool.lookup.sizeOfTx(signed.Hash()) + if !exist { + t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + } + if size != signed.Size() { + t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", + i, j, size, signed.Size()) + } + } + } + if add.check != nil { + if !add.check(pool, signed) { + t.Errorf("test %d, tx %d: custom check failed", i, j) } } + // Verify the pool internals after each addition verifyPoolInternals(t, pool) } verifyPoolInternals(t, pool) @@ -1806,66 +1825,6 @@ func TestAdd(t *testing.T) { } } -// Tests that transactions with legacy sidecars are accepted within the -// conversion window but rejected after it has passed. -func TestAddLegacyBlobTx(t *testing.T) { - testAddLegacyBlobTx(t, true) // conversion window has not yet passed - testAddLegacyBlobTx(t, false) // conversion window passed -} - -func testAddLegacyBlobTx(t *testing.T, accept bool) { - var ( - key1, _ = crypto.GenerateKey() - key2, _ = crypto.GenerateKey() - - addr1 = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = crypto.PubkeyToAddress(key2.PublicKey) - ) - - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - statedb.Commit(0, true, false) - - chain := &testBlockChain{ - config: params.MergedTestChainConfig, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), - statedb: statedb, - } - var timeDiff uint64 - if accept { - timeDiff = uint64(conversionTimeWindow.Seconds()) - 1 - } else { - timeDiff = uint64(conversionTimeWindow.Seconds()) + 1 - } - time := *params.MergedTestChainConfig.OsakaTime + timeDiff - chain.setHeadTime(time) - - pool := New(Config{Datadir: t.TempDir()}, chain, nil) - if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { - t.Fatalf("failed to create blob pool: %v", err) - } - - // Attempt to add legacy blob transactions. - var ( - tx1 = makeMultiBlobTx(0, 1, 1000, 100, 6, 0, key1, types.BlobSidecarVersion0) - tx2 = makeMultiBlobTx(0, 1, 800, 70, 6, 6, key2, types.BlobSidecarVersion0) - txs = []*types.Transaction{tx1, tx2} - ) - errs := pool.Add(txs, true) - for _, err := range errs { - if accept && err != nil { - t.Fatalf("expected tx add to succeed, %v", err) - } - if !accept && err == nil { - t.Fatal("expected tx add to fail") - } - } - verifyPoolInternals(t, pool) - pool.Close() -} - func TestGetBlobs(t *testing.T) { //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) @@ -1952,7 +1911,6 @@ func TestGetBlobs(t *testing.T) { limit int fillRandom bool // Whether to randomly fill some of the requested blobs with unknowns version byte // Blob sidecar version to request - convert bool // Whether to convert version on retrieval }{ { start: 0, limit: 6, @@ -2018,11 +1976,6 @@ func TestGetBlobs(t *testing.T) { start: 0, limit: 18, fillRandom: true, version: types.BlobSidecarVersion1, }, - { - start: 0, limit: 18, fillRandom: true, - version: types.BlobSidecarVersion1, - convert: true, // Convert some version 0 blobs to version 1 while retrieving - }, } for i, c := range cases { var ( @@ -2044,7 +1997,7 @@ func TestGetBlobs(t *testing.T) { filled[len(vhashes)] = struct{}{} vhashes = append(vhashes, testrand.Hash()) } - blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version, c.convert) + blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version) if err != nil { t.Errorf("Unexpected error for case %d, %v", i, err) } @@ -2070,8 +2023,7 @@ func TestGetBlobs(t *testing.T) { // If an item is missing, but shouldn't, error if blobs[j] == nil || proofs[j] == nil { // This is only an error if there was no version mismatch - if c.convert || - (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || + if (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || (c.version == types.BlobSidecarVersion0 && (testBlobIndex < 6 || 12 <= testBlobIndex)) { t.Errorf("tracked blob retrieval failed: item %d, hash %x", j, vhashes[j]) } @@ -2098,185 +2050,6 @@ func TestGetBlobs(t *testing.T) { pool.Close() } -// TestSidecarConversion will verify that after the Osaka fork, all legacy -// sidecars in the pool are successfully convert to v1 sidecars. -func TestSidecarConversion(t *testing.T) { - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) - - // Create a temporary folder for the persistent backend - storage := t.TempDir() - os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - - var ( - preOsakaTxs = make(types.Transactions, 10) - postOsakaTxs = make(types.Transactions, 3) - keys = make([]*ecdsa.PrivateKey, len(preOsakaTxs)+len(postOsakaTxs)) - addrs = make([]common.Address, len(preOsakaTxs)+len(postOsakaTxs)) - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - ) - for i := range keys { - keys[i], _ = crypto.GenerateKey() - addrs[i] = crypto.PubkeyToAddress(keys[i].PublicKey) - statedb.AddBalance(addrs[i], uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - } - for i := range preOsakaTxs { - preOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 2, 0, keys[i], types.BlobSidecarVersion0) - } - for i := range postOsakaTxs { - if i == 0 { - // First has a v0 sidecar. - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion0) - } - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion1) - } - statedb.Commit(0, true, false) - - // Test plan: - // 1) Create a bunch v0 sidecar txs and add to pool before Osaka. - // 2) Pass in new Osaka header to activate the conversion thread. - // 3) Continue adding both v0 and v1 transactions to the pool. - // 4) Verify that as additional blocks come in, transactions involved in the - // migration are correctly discarded. - - config := ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - LondonBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - CancunTime: newUint64(0), - PragueTime: newUint64(0), - OsakaTime: newUint64(1), - BlobScheduleConfig: params.DefaultBlobSchedule, - } - chain := &testBlockChain{ - config: config, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), - statedb: statedb, - blocks: make(map[uint64]*types.Block), - } - - // Create 3 blocks: - // - the current block, before Osaka - // - the first block after Osaka - // - another post-Osaka block with several transactions in it - header0 := chain.CurrentBlock() - header0.Time = 0 - chain.blocks[0] = types.NewBlockWithHeader(header0) - - header1 := chain.CurrentBlock() - header1.Number = big.NewInt(1) - header1.Time = 1 - chain.blocks[1] = types.NewBlockWithHeader(header1) - - header2 := chain.CurrentBlock() - header2.Time = 2 - header2.Number = big.NewInt(2) - - // Make a copy of one of the pre-Osaka transactions and convert it to v1 here - // so that we can add it to the pool later and ensure a duplicate is not added - // by the conversion queue. - tx := preOsakaTxs[len(preOsakaTxs)-1] - sc := *tx.BlobTxSidecar() // copy sidecar - sc.ToV1() - tx.WithBlobTxSidecar(&sc) - - block2 := types.NewBlockWithHeader(header2).WithBody(types.Body{Transactions: append(postOsakaTxs, tx)}) - chain.blocks[2] = block2 - - pool := New(Config{Datadir: storage}, chain, nil) - if err := pool.Init(1, header0, newReserver()); err != nil { - t.Fatalf("failed to create blob pool: %v", err) - } - - errs := pool.Add(preOsakaTxs, true) - for i, err := range errs { - if err != nil { - t.Errorf("failed to insert blob tx from %s: %s", addrs[i], errs[i]) - } - } - - // Kick off migration. - pool.Reset(header0, header1) - - // Add the v0 sidecar tx, but don't block so we can keep doing other stuff - // while it converts the sidecar. - addDone := make(chan struct{}) - go func() { - pool.Add(types.Transactions{postOsakaTxs[0]}, false) - close(addDone) - }() - - // Add the post-Osaka v1 sidecar txs. - errs = pool.Add(postOsakaTxs[1:], false) - for _, err := range errs { - if err != nil { - t.Fatalf("expected tx add to succeed: %v", err) - } - } - - // Wait for the first tx's conversion to complete, then check that all - // transactions added after Osaka can be accounted for in the pool. - <-addDone - pending := pool.Pending(txpool.PendingFilter{BlobTxs: true, BlobVersion: types.BlobSidecarVersion1}) - for _, tx := range postOsakaTxs { - from, _ := pool.signer.Sender(tx) - if len(pending[from]) != 1 || pending[from][0].Hash != tx.Hash() { - t.Fatalf("expected post-Osaka txs to be pending") - } - } - - // Now update the pool with the next block. This should cause the pool to - // clear out the post-Osaka txs since they were included in block 2. Since the - // test blockchain doesn't manage nonces, we'll just do that manually before - // the reset is called. Don't forget about the pre-Osaka transaction we also - // added to block 2! - for i := range postOsakaTxs { - statedb.SetNonce(addrs[len(preOsakaTxs)+i], 1, tracing.NonceChangeEoACall) - } - statedb.SetNonce(addrs[len(preOsakaTxs)-1], 1, tracing.NonceChangeEoACall) - pool.Reset(header1, block2.Header()) - - // Now verify no post-Osaka transactions are tracked by the pool. - for i, tx := range postOsakaTxs { - if pool.Get(tx.Hash()) != nil { - t.Fatalf("expected txs added post-osaka to have been placed in limbo due to inclusion in a block: index %d, hash %s", i, tx.Hash()) - } - } - - // Wait for the pool migration to complete. - <-pool.cQueue.anyBillyConversionDone - - // Verify all transactions in the pool were converted and verify the - // subsequent cell proofs. - count, _ := pool.Stats() - if count != len(preOsakaTxs)-1 { - t.Errorf("expected pending count to match initial tx count: pending=%d, expected=%d", count, len(preOsakaTxs)-1) - } - for addr, acc := range pool.index { - for _, m := range acc { - if m.version != types.BlobSidecarVersion1 { - t.Errorf("expected sidecar to have been converted: from %s, hash %s", addr, m.hash) - } - tx := pool.Get(m.hash) - if tx == nil { - t.Errorf("failed to get tx by hash: %s", m.hash) - } - sc := tx.BlobTxSidecar() - if err := kzg4844.VerifyCellProofs(sc.Blobs, sc.Commitments, sc.Proofs); err != nil { - t.Errorf("failed to verify cell proofs for tx %s after conversion: %s", m.hash, err) - } - } - } - - verifyPoolInternals(t, pool) - - // Launch conversion a second time. - // This is just a sanity check to ensure we can handle it. - pool.Reset(header0, header1) - - pool.Close() -} - // fakeBilly is a billy.Database implementation which just drops data on the floor. type fakeBilly struct { billy.Database @@ -2353,11 +2126,10 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { MinTip: uint256.NewInt(1), BaseFee: chain.basefee, BlobFee: chain.blobfee, + BlobTxs: true, }) if len(p) != int(capacity) { b.Fatalf("have %d want %d", len(p), capacity) } } } - -func newUint64(val uint64) *uint64 { return &val } diff --git a/core/txpool/blobpool/conversion.go b/core/txpool/blobpool/conversion.go deleted file mode 100644 index 95828d83b2..0000000000 --- a/core/txpool/blobpool/conversion.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "errors" - "slices" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -// maxPendingConversionTasks caps the number of pending conversion tasks. This -// prevents excessive memory usage; the worst-case scenario (2k transactions -// with 6 blobs each) would consume approximately 1.5GB of memory. -const maxPendingConversionTasks = 2048 - -// txConvert represents a conversion task with an attached legacy blob transaction. -type txConvert struct { - tx *types.Transaction // Legacy blob transaction - done chan error // Channel for signaling back if the conversion succeeds -} - -// conversionQueue is a dedicated queue for converting legacy blob transactions -// received from the network after the Osaka fork. Since conversion is expensive, -// it is performed in the background by a single thread, ensuring the main Geth -// process is not overloaded. -type conversionQueue struct { - tasks chan *txConvert - startBilly chan func() - quit chan struct{} - closed chan struct{} - - billyQueue []func() - billyTaskDone chan struct{} - - // This channel will be closed when the first billy conversion finishes. - // It's added for unit tests to synchronize with the conversion progress. - anyBillyConversionDone chan struct{} -} - -// newConversionQueue constructs the conversion queue. -func newConversionQueue() *conversionQueue { - q := &conversionQueue{ - tasks: make(chan *txConvert), - startBilly: make(chan func()), - quit: make(chan struct{}), - closed: make(chan struct{}), - anyBillyConversionDone: make(chan struct{}), - } - go q.loop() - return q -} - -// convert accepts a legacy blob transaction with version-0 blobs and queues it -// for conversion. -// -// This function may block for a long time until the transaction is processed. -func (q *conversionQueue) convert(tx *types.Transaction) error { - done := make(chan error, 1) - select { - case q.tasks <- &txConvert{tx: tx, done: done}: - return <-done - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// launchBillyConversion starts a conversion task in the background. -func (q *conversionQueue) launchBillyConversion(fn func()) error { - select { - case q.startBilly <- fn: - return nil - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// close terminates the conversion queue. -func (q *conversionQueue) close() { - select { - case <-q.closed: - return - default: - close(q.quit) - <-q.closed - } -} - -// run converts a batch of legacy blob txs to the new cell proof format. -func (q *conversionQueue) run(tasks []*txConvert, done chan struct{}, interrupt *atomic.Int32) { - defer close(done) - - for _, t := range tasks { - if interrupt != nil && interrupt.Load() != 0 { - t.done <- errors.New("conversion is interrupted") - continue - } - sidecar := t.tx.BlobTxSidecar() - if sidecar == nil { - t.done <- errors.New("tx without sidecar") - continue - } - // Run the conversion, the original sidecar will be mutated in place - start := time.Now() - err := sidecar.ToV1() - t.done <- err - log.Trace("Converted legacy blob tx", "hash", t.tx.Hash(), "err", err, "elapsed", common.PrettyDuration(time.Since(start))) - } -} - -func (q *conversionQueue) loop() { - defer close(q.closed) - - var ( - done chan struct{} // Non-nil if background routine is active - interrupt *atomic.Int32 // Flag to signal conversion interruption - - // The pending tasks for sidecar conversion. We assume the number of legacy - // blob transactions requiring conversion will not be excessive. However, - // a hard cap is applied as a protective measure. - txTasks []*txConvert - - firstBilly = true - ) - - for { - select { - case t := <-q.tasks: - if len(txTasks) >= maxPendingConversionTasks { - t.done <- errors.New("conversion queue is overloaded") - continue - } - txTasks = append(txTasks, t) - - // Launch the background conversion thread if it's idle - if done == nil { - done, interrupt = make(chan struct{}), new(atomic.Int32) - - tasks := slices.Clone(txTasks) - txTasks = txTasks[:0] - go q.run(tasks, done, interrupt) - } - - case <-done: - done, interrupt = nil, nil - - case fn := <-q.startBilly: - q.billyQueue = append(q.billyQueue, fn) - q.runNextBillyTask() - - case <-q.billyTaskDone: - if firstBilly { - close(q.anyBillyConversionDone) - firstBilly = false - } - q.runNextBillyTask() - - case <-q.quit: - if done != nil { - log.Debug("Waiting for blob proof conversion to exit") - interrupt.Store(1) - <-done - } - if q.billyTaskDone != nil { - log.Debug("Waiting for blobpool billy conversion to exit") - <-q.billyTaskDone - } - return - } - } -} - -func (q *conversionQueue) runNextBillyTask() { - if len(q.billyQueue) == 0 { - q.billyTaskDone = nil - return - } - - fn := q.billyQueue[0] - q.billyQueue = append(q.billyQueue[:0], q.billyQueue[1:]...) - - done := make(chan struct{}) - go func() { defer close(done); fn() }() - q.billyTaskDone = done -} diff --git a/core/txpool/blobpool/conversion_test.go b/core/txpool/blobpool/conversion_test.go deleted file mode 100644 index a9fd26dbaf..0000000000 --- a/core/txpool/blobpool/conversion_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "crypto/ecdsa" - "crypto/sha256" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/kzg4844" - "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" -) - -// createV1BlobTx creates a blob transaction with version 1 sidecar for testing. -func createV1BlobTx(nonce uint64, key *ecdsa.PrivateKey) *types.Transaction { - blob := &kzg4844.Blob{byte(nonce)} - commitment, _ := kzg4844.BlobToCommitment(blob) - cellProofs, _ := kzg4844.ComputeCellProofs(blob) - - blobtx := &types.BlobTx{ - ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID), - Nonce: nonce, - GasTipCap: uint256.NewInt(1), - GasFeeCap: uint256.NewInt(1000), - Gas: 21000, - BlobFeeCap: uint256.NewInt(100), - BlobHashes: []common.Hash{kzg4844.CalcBlobHashV1(sha256.New(), &commitment)}, - Value: uint256.NewInt(100), - Sidecar: types.NewBlobTxSidecar(types.BlobSidecarVersion1, []kzg4844.Blob{*blob}, []kzg4844.Commitment{commitment}, cellProofs), - } - return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx) -} - -func TestConversionQueueBasic(t *testing.T) { - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - if err := queue.convert(tx); err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != types.BlobSidecarVersion1 { - t.Errorf("Expected sidecar version to be %d, got %d", types.BlobSidecarVersion1, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueV1BlobTx(t *testing.T) { - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := createV1BlobTx(0, key) - version := tx.BlobTxSidecar().Version - - err := queue.convert(tx) - if err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != version { - t.Errorf("Expected sidecar version to remain %d, got %d", version, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueClosed(t *testing.T) { - queue := newConversionQueue() - - // Close the queue first - queue.close() - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - - err := queue.convert(tx) - if err == nil { - t.Fatal("Expected error when converting on closed queue, got nil") - } -} - -func TestConversionQueueDoubleClose(t *testing.T) { - queue := newConversionQueue() - queue.close() - queue.close() // Should not panic -} diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index 50c40c9d83..36284d6a03 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -20,7 +20,6 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -57,7 +56,7 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) { } // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(config)) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the limbo after fusaka. slotter, err := tryMigrate(config, slotter, datadir) diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go index 874ca85b8c..7607cd487a 100644 --- a/core/txpool/blobpool/lookup.go +++ b/core/txpool/blobpool/lookup.go @@ -110,13 +110,3 @@ func (l *lookup) untrack(tx *blobTxMeta) { } } } - -// update updates the transaction index. It should only be used in the conversion. -func (l *lookup) update(hash common.Hash, id uint64, size uint64) bool { - meta, exists := l.txIndex[hash] - if !exists { - return false - } - meta.id, meta.size = id, size - return true -} diff --git a/core/txpool/blobpool/slotter.go b/core/txpool/blobpool/slotter.go index 9b793e366c..3399361e55 100644 --- a/core/txpool/blobpool/slotter.go +++ b/core/txpool/blobpool/slotter.go @@ -17,7 +17,6 @@ package blobpool import ( - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/params" "github.com/holiman/billy" ) @@ -42,7 +41,7 @@ func tryMigrate(config *params.ChainConfig, slotter billy.SlotSizeFn, datadir st // If the version found is less than the currently configured store version, // perform a migration then write the updated version of the store. if version < storeVersion { - newSlotter := newSlotterEIP7594(eip4844.LatestMaxBlobsPerBlock(config)) + newSlotter := newSlotterEIP7594(params.BlobTxMaxBlobs) if err := billy.Migrate(billy.Options{Path: datadir, Repair: true}, slotter, newSlotter); err != nil { return nil, err } @@ -54,7 +53,7 @@ func tryMigrate(config *params.ChainConfig, slotter billy.SlotSizeFn, datadir st store.Close() } // Set the slotter to the format now that the Osaka is active. - slotter = newSlotterEIP7594(eip4844.LatestMaxBlobsPerBlock(config)) + slotter = newSlotterEIP7594(params.BlobTxMaxBlobs) } return slotter, nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 9da0bc2121..37c72682fc 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -114,6 +114,9 @@ var ( queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) + pendingAddrsGauge = metrics.NewRegisteredGauge("txpool/pending/accounts", nil) + queuedAddrsGauge = metrics.NewRegisteredGauge("txpool/queued/accounts", nil) + reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) ) @@ -148,7 +151,7 @@ type Config struct { AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts - Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + Lifetime time.Duration // Maximum amount of time an account can remain stale in the non-executable pool } // DefaultConfig contains the default configurations for the transaction pool. @@ -288,7 +291,12 @@ func New(config Config, chain BlockChain) *LegacyPool { // Filter returns whether the given transaction can be consumed by the legacy // pool, specifically, whether it is a Legacy, AccessList or Dynamic transaction. func (pool *LegacyPool) Filter(tx *types.Transaction) bool { - switch tx.Type() { + return pool.FilterType(tx.Type()) +} + +// FilterType returns whether the legacy pool supports the given transaction type. +func (pool *LegacyPool) FilterType(kind byte) bool { + switch kind { case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.SetCodeTxType: return true default: @@ -770,7 +778,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) - // Successful promotion, bump the heartbeat + // Successful replacement. If needed, bump the heartbeat giving more time to queued txs. pool.queue.bump(from) return old != nil, nil } @@ -839,6 +847,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newList(true) + pendingAddrsGauge.Inc(1) } list := pool.pending[addr] @@ -862,7 +871,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) - // Successful promotion, bump the heartbeat + // Successful promotion, bump the heartbeat, giving more time to queued txs. pool.queue.bump(addr) return true } @@ -899,8 +908,8 @@ func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( - errs = make([]error, len(txs)) - news = make([]*types.Transaction, 0, len(txs)) + hasValid bool + errs = make([]error, len(txs)) ) for i, tx := range txs { // If the transaction is known, pre-set the error slot @@ -918,26 +927,17 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { invalidTxMeter.Mark(1) continue } - // Accumulate all unknown transactions for deeper processing - news = append(news, tx) + hasValid = true } - if len(news) == 0 { + if !hasValid { return errs } // Process all the new transaction and merge any errors into the original slice pool.mu.Lock() - newErrs, dirtyAddrs := pool.addTxsLocked(news) + dirtyAddrs := pool.addTxsLocked(txs, errs) pool.mu.Unlock() - var nilSlot = 0 - for _, err := range newErrs { - for errs[nilSlot] != nil { - nilSlot++ - } - errs[nilSlot] = err - nilSlot++ - } // Reorg the pool internals if needed and return done := pool.requestPromoteExecutables(dirtyAddrs) if sync { @@ -948,14 +948,19 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // addTxsLocked attempts to queue a batch of transactions if they are valid. // The transaction pool lock must be held. -// Returns the error for each tx, and the set of accounts that might became promotable. -func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction) ([]error, *accountSet) { +// Sets the error for each tx, and the set of accounts that might became promotable. +// We only try to add txs that have no error set in the errs slice. +// If adding the transaction returns an error, we set the error in the errs slice. +// Requires len(txs) == len(errs). +func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction, errs []error) *accountSet { var ( dirty = newAccountSet(pool.signer) - errs = make([]error, len(txs)) valid int64 ) for i, tx := range txs { + if errs[i] != nil { + continue + } replaced, err := pool.add(tx) errs[i] = err if err == nil { @@ -966,7 +971,7 @@ func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction) ([]error, *accoun } } validTxMeter.Mark(valid) - return errs, dirty + return dirty } // Status returns the status (unknown/pending/queued) of a batch of transactions @@ -1078,6 +1083,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo // If no more pending transactions are left, remove the list if pending.Empty() { delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) } // Postpone any invalidated transactions for _, tx := range invalids { @@ -1387,7 +1393,7 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { // Inject any transactions discarded due to reorgs log.Debug("Reinjecting stale transactions", "count", len(reinject)) core.SenderCacher().Recover(pool.signer, reinject) - pool.addTxsLocked(reinject) + pool.addTxsLocked(reinject, make([]error, len(reinject))) } // promoteExecutables moves transactions that have become processable from the @@ -1558,6 +1564,7 @@ func (pool *LegacyPool) demoteUnexecutables() { // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(hash, tx, false) } + pool.priced.Removed(len(olds) + len(drops)) pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) // If there's a gap in front, alert (should never happen) and postpone all transactions @@ -1575,6 +1582,7 @@ func (pool *LegacyPool) demoteUnexecutables() { // Delete the entire pending entry if it became empty. if list.Empty() { delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) if _, ok := pool.queue.get(addr); !ok { pool.reserver.Release(addr) } @@ -1834,6 +1842,13 @@ func (pool *LegacyPool) Clear() { pool.pending = make(map[common.Address]*list) pool.queue = newQueue(pool.config, pool.signer) pool.pendingNonces = newNoncer(pool.currentState) + + // Reset gauges + pendingGauge.Update(0) + queuedGauge.Update(0) + slotsGauge.Update(0) + pendingAddrsGauge.Update(0) + queuedAddrsGauge.Update(0) } // HasPendingAuth returns a flag indicating whether there are pending diff --git a/core/txpool/legacypool/queue.go b/core/txpool/legacypool/queue.go index a889debe37..35beaded12 100644 --- a/core/txpool/legacypool/queue.go +++ b/core/txpool/legacypool/queue.go @@ -88,8 +88,12 @@ func (q *queue) get(addr common.Address) (*list, bool) { return l, ok } +// bump updates the heartbeat for the given account address. +// If the address is unknown, the call is a no-op. func (q *queue) bump(addr common.Address) { - q.beats[addr] = time.Now() + if _, ok := q.beats[addr]; ok { + q.beats[addr] = time.Now() + } } func (q *queue) addresses() []common.Address { @@ -114,6 +118,7 @@ func (q *queue) remove(addr common.Address, tx *types.Transaction) { if future.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) } } } @@ -123,6 +128,7 @@ func (q *queue) add(tx *types.Transaction) (*common.Hash, error) { from, _ := types.Sender(q.signer, tx) // already validated if q.queued[from] == nil { q.queued[from] = newList(false) + queuedAddrsGauge.Inc(1) } inserted, old := q.queued[from].Add(tx, q.config.PriceBump) if !inserted { @@ -200,6 +206,7 @@ func (q *queue) promoteExecutables(accounts []common.Address, gasLimit uint64, c if list.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) removedAddresses = append(removedAddresses, addr) } } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 519ae7b989..db099ddf98 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -100,6 +100,9 @@ type SubPool interface { // to this particular subpool. Filter(tx *types.Transaction) bool + // FilterType returns whether the subpool supports the given transaction type. + FilterType(kind byte) bool + // Init sets the base parameters of the subpool, allowing it to load any saved // transactions from disk and also permitting internal maintenance routines to // start up. diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 437861efca..a314a83f1b 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -489,3 +489,14 @@ func (p *TxPool) Clear() { subpool.Clear() } } + +// FilterType returns whether a transaction with the given type is supported +// (can be added) by the pool. +func (p *TxPool) FilterType(kind byte) bool { + for _, subpool := range p.subpools { + if subpool.FilterType(kind) { + return true + } + } + return false +} diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 773de530e9..aa8ce27485 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -131,7 +131,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } // Ensure the transaction can cover floor data gas. - if opts.Config.IsPrague(head.Number, head.Time, arbosVersion) { + if rules.IsPrague { floorDataGas, err := core.FloorDataGas(tx.Data()) if err != nil { return err @@ -161,6 +161,15 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO if sidecar == nil { return errors.New("missing sidecar in blob transaction") } + // Ensure the sidecar is constructed with the correct version, consistent + // with the current fork. + version := types.BlobSidecarVersion0 + if opts.Config.IsOsaka(head.Number, head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) { + version = types.BlobSidecarVersion1 + } + if sidecar.Version != version { + return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version) + } // Ensure the blob fee cap satisfies the minimum blob gas price if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) diff --git a/core/types.go b/core/types.go index bed20802ab..87bbfcff58 100644 --- a/core/types.go +++ b/core/types.go @@ -17,6 +17,7 @@ package core import ( + "context" "sync/atomic" "github.com/ethereum/go-ethereum/core/state" @@ -48,7 +49,7 @@ type Processor interface { // Process processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb and applying any rewards to both // the processor (coinbase) and any included uncles. - Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) + Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) } // ProcessResult contains the values computed by Process. diff --git a/core/types/block.go b/core/types/block.go index b5b6468a13..c52c05a4c7 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-verkle" ) // A BlockNonce is a 64-bit hash which proves (combined with the @@ -61,13 +60,6 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } -// ExecutionWitness represents the witness + proof used in a verkle context, -// to provide the ability to execute a block statelessly. -type ExecutionWitness struct { - StateDiff verkle.StateDiff `json:"stateDiff"` - VerkleProof *verkle.VerkleProof `json:"verkleProof"` -} - //go:generate go run github.com/fjl/gencodec -type Header -field-override headerMarshaling -out gen_header_json.go //go:generate go run ../../rlp/rlpgen -type Header -out gen_header_rlp.go @@ -209,11 +201,6 @@ type Block struct { transactions Transactions withdrawals Withdrawals - // witness is not an encoded part of the block body. - // It is held in Block in order for easy relaying to the places - // that process it. - witness *ExecutionWitness - // caches hash atomic.Pointer[common.Hash] size atomic.Uint64 @@ -429,9 +416,6 @@ func (b *Block) BlobGasUsed() *uint64 { return blobGasUsed } -// ExecutionWitness returns the verkle execution witneess + proof for a block -func (b *Block) ExecutionWitness() *ExecutionWitness { return b.witness } - // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previously cached value. func (b *Block) Size() uint64 { @@ -494,7 +478,6 @@ func (b *Block) WithSeal(header *Header) *Block { transactions: b.transactions, uncles: b.uncles, withdrawals: b.withdrawals, - witness: b.witness, } } @@ -506,7 +489,6 @@ func (b *Block) WithBody(body Body) *Block { transactions: slices.Clone(body.Transactions), uncles: make([]*Header, len(body.Uncles)), withdrawals: slices.Clone(body.Withdrawals), - witness: b.witness, } for i := range body.Uncles { block.uncles[i] = CopyHeader(body.Uncles[i]) @@ -514,16 +496,6 @@ func (b *Block) WithBody(body Body) *Block { return block } -func (b *Block) WithWitness(witness *ExecutionWitness) *Block { - return &Block{ - header: b.header, - transactions: b.transactions, - uncles: b.uncles, - withdrawals: b.withdrawals, - witness: witness, - } -} - // Hash returns the keccak256 hash of b's header. // The hash is computed on the first call and cached thereafter. func (b *Block) Hash() common.Hash { diff --git a/core/types/log.go b/core/types/log.go index 84dc051f82..106a9b96e2 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -68,7 +68,7 @@ type logMarshaling struct { // FilterLogs creates a slice of logs matching the given criteria. func FilterLogs(logs []*Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*Log { - var check = func(log *Log) bool { + check := func(log *Log) bool { if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber { return false } diff --git a/core/types/receipt.go b/core/types/receipt.go index 0ba5034c68..9ef45b566f 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -539,3 +539,33 @@ func EncodeBlockReceiptLists(receipts []Receipts) []rlp.RawValue { } return result } + +// SlimReceipt is a wrapper around a Receipt with RLP serialization that omits +// the Bloom field and includes the tx type. Used for era files. +type SlimReceipt Receipt + +type slimReceiptRLP struct { + Type uint8 + StatusEncoding []byte + CumulativeGasUsed uint64 + Logs []*Log +} + +// EncodeRLP implements rlp.Encoder, encoding the receipt as +// [tx-type, post-state-or-status, cumulative-gas, logs]. +func (r *SlimReceipt) EncodeRLP(w io.Writer) error { + data := &slimReceiptRLP{r.Type, (*Receipt)(r).statusEncoding(), r.CumulativeGasUsed, r.Logs} + return rlp.Encode(w, data) +} + +// DecodeRLP implements rlp.Decoder. +func (r *SlimReceipt) DecodeRLP(s *rlp.Stream) error { + var data slimReceiptRLP + if err := s.Decode(&data); err != nil { + return err + } + r.Type = data.Type + r.CumulativeGasUsed = data.CumulativeGasUsed + r.Logs = data.Logs + return (*Receipt)(r).setStatus(data.StatusEncoding) +} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 8408d05ce4..87342ad27f 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -515,6 +515,45 @@ func TestReceiptUnmarshalBinary(t *testing.T) { } } +func TestSlimReceiptEncodingDecoding(t *testing.T) { + tests := []*Receipt{ + legacyReceipt, + accessListReceipt, + eip1559Receipt, + { + Type: BlobTxType, + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 100, + Logs: []*Log{}, + }, + } + for i, want := range tests { + enc, err := rlp.EncodeToBytes((*SlimReceipt)(want)) + if err != nil { + t.Fatalf("test %d: encode error: %v", i, err) + } + got := new(SlimReceipt) + if err := rlp.DecodeBytes(enc, got); err != nil { + t.Fatalf("test %d: decode error: %v", i, err) + } + if got.Type != want.Type { + t.Errorf("test %d: Type mismatch: got %d, want %d", i, got.Type, want.Type) + } + if got.Status != want.Status { + t.Errorf("test %d: Status mismatch: got %d, want %d", i, got.Status, want.Status) + } + if !bytes.Equal(got.PostState, want.PostState) { + t.Errorf("test %d: PostState mismatch: got %x, want %x", i, got.PostState, want.PostState) + } + if got.CumulativeGasUsed != want.CumulativeGasUsed { + t.Errorf("test %d: CumulativeGasUsed mismatch: got %d, want %d", i, got.CumulativeGasUsed, want.CumulativeGasUsed) + } + if len(got.Logs) != len(want.Logs) { + t.Errorf("test %d: Logs length mismatch: got %d, want %d", i, len(got.Logs), len(want.Logs)) + } + } +} + func clearComputedFieldsOnReceipts(receipts []*Receipt) []*Receipt { r := make([]*Receipt, len(receipts)) for i, receipt := range receipts { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index cbe12139eb..893a200d5c 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -299,7 +299,10 @@ func (s *modernSigner) SignatureValues(tx *Transaction, sig []byte) (R, S, V *bi if tx.inner.chainID().Sign() != 0 && tx.inner.chainID().Cmp(s.chainID) != 0 { return nil, nil, nil, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.inner.chainID(), s.chainID) } - R, S, _ = decodeSignature(sig) + R, S, _, err = decodeSignature(sig) + if err != nil { + return nil, nil, nil, err + } V = big.NewInt(int64(sig[64])) return R, S, V, nil } @@ -344,7 +347,7 @@ func NewEIP2930Signer(chainId *big.Int) Signer { // are replay-protected as well as unprotected homestead transactions. // Deprecated: always use the Signer interface type type EIP155Signer struct { - chainId, chainIdMul *big.Int + chainId *big.Int } func NewEIP155Signer(chainId *big.Int) EIP155Signer { @@ -352,8 +355,7 @@ func NewEIP155Signer(chainId *big.Int) EIP155Signer { chainId = new(big.Int) } return EIP155Signer{ - chainId: chainId, - chainIdMul: new(big.Int).Mul(chainId, big.NewInt(2)), + chainId: chainId, } } @@ -379,7 +381,8 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) } V, R, S := tx.RawSignatureValues() - V = new(big.Int).Sub(V, s.chainIdMul) + V = new(big.Int).Sub(V, s.chainId) + V = new(big.Int).Sub(V, s.chainId) V.Sub(V, big8) return recoverPlain(s.Hash(tx), R, S, V, true) } @@ -390,10 +393,14 @@ func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big if tx.Type() != LegacyTxType { return nil, nil, nil, ErrTxTypeNotSupported } - R, S, V = decodeSignature(sig) + R, S, V, err = decodeSignature(sig) + if err != nil { + return nil, nil, nil, err + } if s.chainId.Sign() != 0 { V = big.NewInt(int64(sig[64] + 35)) - V.Add(V, s.chainIdMul) + V.Add(V, s.chainId) + V.Add(V, s.chainId) } return R, S, V, nil } @@ -459,8 +466,8 @@ func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v * if tx.Type() != LegacyTxType { return nil, nil, nil, ErrTxTypeNotSupported } - r, s, v = decodeSignature(sig) - return r, s, v, nil + r, s, v, err = decodeSignature(sig) + return r, s, v, err } // Hash returns the hash to be signed by the sender. @@ -476,14 +483,14 @@ func (fs FrontierSigner) Hash(tx *Transaction) common.Hash { }) } -func decodeSignature(sig []byte) (r, s, v *big.Int) { +func decodeSignature(sig []byte) (r, s, v *big.Int, err error) { if len(sig) != crypto.SignatureLength { - panic(fmt.Sprintf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength)) + return nil, nil, nil, fmt.Errorf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength) } r = new(big.Int).SetBytes(sig[:32]) s = new(big.Int).SetBytes(sig[32:64]) v = new(big.Int).SetBytes([]byte{sig[64] + 27}) - return r, s, v + return r, s, v, nil } func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (common.Address, error) { diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 02a65fda13..252593e87b 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -200,3 +200,28 @@ func Benchmark_modernSigner_Equal(b *testing.B) { } } } + +func TestSignatureValuesError(t *testing.T) { + // 1. Setup a valid transaction + tx := NewTransaction(0, common.Address{}, big.NewInt(0), 0, big.NewInt(0), nil) + signer := HomesteadSigner{} + + // 2. Call WithSignature with invalid length sig (not 65 bytes) + invalidSig := make([]byte, 64) + + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Panicked for invalid signature length, expected error: %v", r) + } + }() + _, err := tx.WithSignature(signer, invalidSig) + if err == nil { + t.Fatal("Expected error for invalid signature length, got nil") + } else { + // This is just a sanity check to ensure we got an error, + // the exact error message is verified in unit tests elsewhere if needed. + t.Logf("Got expected error: %v", err) + } + }() +} diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index dee4d1fc2d..e7a2b91601 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -118,8 +118,8 @@ func (sc *BlobTxSidecar) ToV1() error { } if sc.Version == BlobSidecarVersion0 { proofs := make([]kzg4844.Proof, 0, len(sc.Blobs)*kzg4844.CellProofsPerBlob) - for _, blob := range sc.Blobs { - cellProofs, err := kzg4844.ComputeCellProofs(&blob) + for i := range sc.Blobs { + cellProofs, err := kzg4844.ComputeCellProofs(&sc.Blobs[i]) if err != nil { return err } diff --git a/core/types/tx_setcode.go b/core/types/tx_setcode.go index f2281d4ae7..9487c9cc81 100644 --- a/core/types/tx_setcode.go +++ b/core/types/tx_setcode.go @@ -94,7 +94,10 @@ func SignSetCode(prv *ecdsa.PrivateKey, auth SetCodeAuthorization) (SetCodeAutho if err != nil { return SetCodeAuthorization{}, err } - r, s, _ := decodeSignature(sig) + r, s, _, err := decodeSignature(sig) + if err != nil { + return SetCodeAuthorization{}, err + } return SetCodeAuthorization{ ChainID: auth.ChainID, Address: auth.Address, diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go deleted file mode 100644 index d8486e2dc9..0000000000 --- a/core/verkle_witness_test.go +++ /dev/null @@ -1,1107 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "math/big" - "slices" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -var ( - testVerkleChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiTime: u64(0), - VerkleTime: u64(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - // TODO uncomment when proof generation is merged - // ProofInBlocks: true, - } - testKaustinenLikeChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(69420), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiTime: u64(0), - VerkleTime: u64(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - } -) - -func TestProcessVerkle(t *testing.T) { - var ( - code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) - intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) - // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness - // will not contain that copied data. - // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 - codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) - intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) - signer = types.LatestSigner(testVerkleChainConfig) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - gspec = &Genesis{ - Config: testVerkleChainConfig, - Alloc: GenesisAlloc{ - coinbase: { - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } - ) - // Verkle trees use the snapshot, which must be enabled before the - // data is saved into the tree+database. - // genesis := gspec.MustCommit(bcdb, triedb) - options := DefaultConfig().WithStateScheme(rawdb.PathScheme) - options.SnapshotLimit = 0 - blockchain, _ := NewBlockChain(bcdb, nil, gspec, beacon.New(ethash.NewFaker()), options) - defer blockchain.Stop() - - txCost1 := params.TxGas - txCost2 := params.TxGas - contractCreationCost := intrinsicContractCreationGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* creation with value */ - 739 /* execution costs */ - codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (tx) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at pc=0x20) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at PC=0x121) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ - 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ - uint64(4844) /* execution costs */ - blockGasUsagesExpected := []uint64{ - txCost1*2 + txCost2, - txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, - } - _, _, chain, _, proofs, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) - tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - - // Add two contract creations in block #2 - if i == 1 { - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 6, - Value: big.NewInt(16), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: code, - }) - gen.AddTx(tx) - - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 7, - Value: big.NewInt(0), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: codeWithExtCodeCopy, - }) - gen.AddTx(tx) - } - }) - - // Check proof for both blocks - err := verkle.Verify(proofs[0], gspec.ToBlock().Root().Bytes(), chain[0].Root().Bytes(), statediffs[0]) - if err != nil { - t.Fatal(err) - } - err = verkle.Verify(proofs[1], chain[0].Root().Bytes(), chain[1].Root().Bytes(), statediffs[1]) - if err != nil { - t.Fatal(err) - } - - t.Log("verified verkle proof, inserting blocks into the chain") - - for i, b := range chain { - fmt.Printf("%d %x\n", i, b.Root()) - } - endnum, err := blockchain.InsertChain(chain) - if err != nil { - t.Fatalf("block %d imported with error: %v", endnum, err) - } - - for i := range 2 { - b := blockchain.GetBlockByNumber(uint64(i) + 1) - if b == nil { - t.Fatalf("expected block %d to be present in chain", i+1) - } - if b.Hash() != chain[i].Hash() { - t.Fatalf("block #%d not found at expected height", b.NumberU64()) - } - if b.GasUsed() != blockGasUsagesExpected[i] { - t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed()) - } - } -} - -func TestProcessParentBlockHash(t *testing.T) { - // This test uses blocks where, - // block 1 parent hash is 0x0100.... - // block 2 parent hash is 0x0200.... - // etc - checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { - statedb.SetNonce(params.HistoryStorageAddress, 1, tracing.NonceChangeUnspecified) - statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode, tracing.CodeChangeUnspecified) - // Process n blocks, from 1 .. num - var num = 2 - for i := 1; i <= num; i++ { - header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} - chainConfig := params.MergedTestChainConfig - if isVerkle { - chainConfig = testVerkleChainConfig - } - vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm) - } - // Read block hashes for block 0 .. num-1 - for i := 0; i < num; i++ { - have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} - if have != want { - t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) - } - } - } - t.Run("MPT", func(t *testing.T) { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - checkBlockHashes(statedb, false) - }) - t.Run("Verkle", func(t *testing.T) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) - checkBlockHashes(statedb, true) - }) -} - -// getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' -func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { - ringIndex := number % params.HistoryServeWindow - var key common.Hash - binary.BigEndian.PutUint64(key[24:], ringIndex) - if isVerkle { - return statedb.GetState(params.HistoryStorageAddress, key) - } - return statedb.GetState(params.HistoryStorageAddress, key) -} - -// TestProcessVerkleInvalidContractCreation checks for several modes of contract creation failures -func TestProcessVerkleInvalidContractCreation(t *testing.T) { - var ( - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(testKaustinenLikeChainConfig) - ) - // slightly modify it to suit the live txs from the testnet - gspec.Alloc[account2] = types.Account{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 1, - } - - // Create two blocks that reproduce what is happening on kaustinen. - // - The first block contains two failing contract creation transactions, that - // write to storage before they revert. - // - // - The second block contains a single failing contract creation transaction, - // that fails right off the bat. - genesisH, _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - for _, rlpData := range []string{ - // SSTORE at slot 41 and reverts - "f8d48084479c2c18830186a08080b8806000602955bda3f9600060ca55600060695523b360006039551983576000601255b0620c2fde2c592ac2600060bc55e0ac6000606455a63e22600060e655eb607e605c5360a2605d5360c7605e53601d605f5360eb606053606b606153608e60625360816063536079606453601e60655360fc60665360b7606753608b60685383021e7ca0cc20c65a97d2e526b8ec0f4266e8b01bdcde43b9aeb59d8bfb44e8eb8119c109a07a8e751813ae1b2ce734960dbc39a4f954917d7822a2c5d1dca18b06c584131f", - // SSTORE at slot 133 and reverts - "02f8db83010f2c01843b9aca0084479c2c18830186a08080b88060006085553fad6000600a55600060565555600060b55506600060cf557f1b8b38183e7bd1bdfaa7123c5a4976e54cce0e42049d841411978fd3595e25c66019527f0538943712953cf08900aae40222a40b2d5a4ac8075ad8cf0870e2be307edbb96039527f9f3174ff85024747041ae7a611acffb987c513c088d90ab288aec080a0cd6ac65ce2cb0a912371f6b5a551ba8caffc22ec55ad4d3cb53de41d05eb77b6a02e0dfe8513dfa6ec7bfd7eda6f5c0dac21b39b982436045e128cec46cfd3f960", - // this one is a simple transfer that succeeds, necessary to get the correct nonce in the other block. - "f8e80184479c2c18830186a094bbbbde4ca27f83fc18aa108170547ff57675936a80b8807ff71f7c15faadb969a76a5f54a81a0117e1e743cb7f24e378eda28442ea4c6eb6604a527fb5409e5718d44e23bfffac926e5ea726067f772772e7e19446acba0c853f62f5606a526020608a536088608b536039608c536004608d5360af608e537f7f7675d9f210e0a61564e6d11e7cd75f5bc9009ac9f6b94a0fc63035441a83021e7ba04a4a172d81ebb02847829b76a387ac09749c8b65668083699abe20c887fb9efca07c5b1a990702ec7b31a5e8e3935cd9a77649f8c25a84131229e24ab61aec6093", - } { - var tx = new(types.Transaction) - if err := tx.UnmarshalBinary(common.Hex2Bytes(rlpData)); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - } else { - var tx = new(types.Transaction) - // immediately reverts - if err := tx.UnmarshalBinary(common.Hex2Bytes("01f8d683010f2c028443ad7d0e830186a08080b880b00e7fa3c849dce891cce5fae8a4c46cbb313d6aec0c0ffe7863e05fb7b22d4807674c6055527ffbfcb0938f3e18f7937aa8fa95d880afebd5c4cec0d85186095832d03c85cf8a60755260ab60955360cf6096536066609753606e60985360fa609953609e609a53608e609b536024609c5360f6609d536072609e5360a4609fc080a08fc6f7101f292ff1fb0de8ac69c2d320fbb23bfe61cf327173786ea5daee6e37a044c42d91838ef06646294bf4f9835588aee66243b16a66a2da37641fae4c045f")); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - }) - - tx1ContractAddress := crypto.CreateAddress(account1, 0) - tx1ContractStem := utils.GetTreeKey(tx1ContractAddress[:], uint256.NewInt(0), 105) - tx1ContractStem = tx1ContractStem[:31] - - tx2ContractAddress := crypto.CreateAddress(account2, 1) - tx2SlotKey := [32]byte{} - tx2SlotKey[31] = 133 - tx2ContractStem := utils.StorageSlotKey(tx2ContractAddress[:], tx2SlotKey[:]) - tx2ContractStem = tx2ContractStem[:31] - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - // Check that the witness contains what we expect: a storage entry for each of the two contract - // creations that failed: one at 133 for the 2nd tx, and one at 105 for the first tx. - for _, stemStateDiff := range statediffs[0] { - // Check that the slot number 133, which is overflowing the account header, - // is present. Note that the offset of the 2nd group (first group after the - // header) is skipping the first 64 values, hence we still have an offset - // of 133, and not 133 - 64. - if bytes.Equal(stemStateDiff.Stem[:], tx2ContractStem[:]) { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 133 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - if suffixDiff.CurrentValue != nil { - t.Fatalf("invalid prestate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.CurrentValue) - } - if suffixDiff.NewValue != nil { - t.Fatalf("invalid poststate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.NewValue) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], tx1ContractStem) { - // For this contract creation, check that only the account header and storage slot 41 - // are found in the witness. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 105 && suffixDiff.Suffix != 0 && suffixDiff.Suffix != 1 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // Check the eip 2935 group of leaves. - // Check that only one leaf was accessed, and is present in the witness. - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - // Check that this leaf is the first storage slot - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the prestate value is nil and that the poststate value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - // For all other entries present in the witness, check that nothing beyond - // the account header was accessed. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } - - // Check that no account has a value above 4 in the 2nd block as no storage nor - // code should make it to the witness. - for _, stemStateDiff := range statediffs[1] { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract at block #2: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 65 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract at block #2: %d != 65", stemStateDiff.SuffixDiffs[0].Suffix) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("missing post state value for BLOCKHASH contract at block #2") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { - t.Fatalf("invalid post state value for BLOCKHASH contract at block #2: %x != %x", chain[0].Hash(), (*stemStateDiff.SuffixDiffs[0].NewValue)[:]) - } - } else if suffixDiff.Suffix > 4 { - t.Fatalf("invalid suffix diff found for %x in block #2: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } -} - -func verkleTestGenesis(config *params.ChainConfig) *Genesis { - var ( - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - ) - return &Genesis{ - Config: config, - Alloc: GenesisAlloc{ - coinbase: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account1: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account2: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 3, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } -} - -// TestProcessVerkleContractWithEmptyCode checks that the witness contains all valid -// entries, if the initcode returns an empty code. -func TestProcessVerkleContractWithEmptyCode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - gspec := verkleTestGenesis(&config) - - genesisH, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - var tx types.Transaction - // a transaction that does some PUSH1n but returns a 0-sized contract - txpayload := common.Hex2Bytes("02f8db83010f2d03843b9aca008444cf6a05830186a08080b8807fdfbbb59f2371a76485ce557fd0de00c298d3ede52a3eab56d35af674eb49ec5860335260826053536001605453604c60555360f3605653606060575360446058536096605953600c605a5360df605b5360f3605c5360fb605d53600c605e53609a605f53607f60605360fe606153603d60625360f4606353604b60645360cac001a0486b6dc55b8a311568b7239a2cae1d77e7446dba71df61eaafd53f73820a138fa010bd48a45e56133ac4c5645142c2ea48950d40eb35050e9510b6bad9e15c5865") - if err := tx.UnmarshalBinary(txpayload); err != nil { - t.Fatal(err) - } - gen.AddTx(&tx) - }) - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - for _, stemStateDiff := range statediffs[0] { - // Handle the case of the history contract: make sure only the correct - // slots are added to the witness. - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the "current value" is nil and that the new value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - // if d8898012c484fb48610ecb7963886339207dab004bce968b007b616ffa18e0 shows up, it means that the PUSHn - // in the transaction above added entries into the witness, when they should not have since they are - // part of a contract deployment. - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } -} - -// TestProcessVerkleExtCodeHashOpcode verifies that calling EXTCODEHASH on another -// deployed contract, creates all the right entries in the witness. -func TestProcessVerkleExtCodeHashOpcode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - dummyContract := []byte{ - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH1), 42, - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - dummyContractAddr := crypto.CreateAddress(deployer, 0) - - // contract that calls EXTCODEHASH on the dummy contract - extCodeHashContract := []byte{ - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH20), - 0x3a, 0x22, 0x0f, 0x35, 0x12, 0x52, 0x08, 0x9d, 0x38, 0x5b, 0x29, 0xbe, 0xca, 0x14, 0xe2, 0x7f, 0x20, 0x4c, 0x29, 0x6a, - byte(vm.EXTCODEHASH), - } - extCodeHashContractAddr := crypto.CreateAddress(deployer, 1) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create dummy contract. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: dummyContract, - }) - gen.AddTx(tx) - - // Create contract with EXTCODEHASH opcode. - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 1, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: extCodeHashContract}) - gen.AddTx(tx) - } else { - tx, _ := types.SignTx(types.NewTransaction(2, extCodeHashContractAddr, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - contractKeccakTreeKey := utils.CodeHashKey(dummyContractAddr[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], contractKeccakTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - codeHashStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - // Check location of code hash was accessed - if codeHashStateDiff.Suffix != utils.CodeHashLeafKey { - t.Fatalf("code hash invalid suffix") - } - // check the code hash wasn't present in the prestate, as - // the contract was deployed in this block. - if codeHashStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - // check the poststate value corresponds to the code hash - // of the deployed contract. - expCodeHash := crypto.Keccak256Hash(dummyContract[12:]) - if *codeHashStateDiff.CurrentValue != expCodeHash { - t.Fatalf("codeHash.CurrentValue unexpected code hash") - } - if codeHashStateDiff.NewValue != nil { - t.Fatalf("codeHash.NewValue must be nil") - } -} - -// TestProcessVerkleBalanceOpcode checks that calling balance -// on another contract will add the correct entries to the witness. -func TestProcessVerkleBalanceOpcode(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - txData := slices.Concat( - []byte{byte(vm.PUSH20)}, - common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d").Bytes(), - []byte{byte(vm.BALANCE)}) - - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: txData}) - gen.AddTx(tx) - }) - - account2BalanceTreeKey := utils.BasicDataKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], account2BalanceTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - var zero [32]byte - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("invalid suffix diff") - } - // check the prestate balance wasn't 0 or missing - if balanceStateDiff.CurrentValue == nil || *balanceStateDiff.CurrentValue == zero { - t.Fatalf("invalid current value %v", *balanceStateDiff.CurrentValue) - } - // check that the poststate witness value for the balance is nil, - // meaning that it didn't get updated. - if balanceStateDiff.NewValue != nil { - t.Fatalf("invalid new value") - } -} - -// TestProcessVerkleSelfDestructInSeparateTx controls the contents of the witness after -// a non-eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSeparateTx(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // runtime code: selfdestruct ( 0x6177843db3138ae69679A54b95cf345ED759450d ) - runtimeCode := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - - //The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a previous transaction. - selfDestructContract := slices.Concat([]byte{ - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), // Codecopy( to-offset: 0, code offset: 12, length: 22 ) - - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), // Return ( 0 : len(runtimecode) - }, - runtimeCode) - - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create selfdestruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - var zero [32]byte - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatalf("the pre-state balance before self-destruct must be %x, got %x", oldBalance, *balanceStateDiff.CurrentValue) - } - - // The new balance must be 0. - if !bytes.Equal((*balanceStateDiff.NewValue)[utils.BasicDataBalanceOffset:], zero[utils.BasicDataBalanceOffset:]) { - t.Fatalf("the post-state balance after self-destruct must be 0") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42, got %d-%d=%d", postStateBalance, preStateBalance, postStateBalance-preStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSameTx controls the contents of the witness after -// a eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSameTx(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in **the same** transaction sending the remaining - // balance to an external (i.e: not itself) account. - - selfDestructContract := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - if balanceStateDiff.CurrentValue != nil { - t.Fatalf("the pre-state balance before must be nil, since the contract didn't exist") - } - - if balanceStateDiff.NewValue != nil { - t.Fatalf("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42. got %d", postStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in a different tx than the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a *previous* transaction sending the remaining - // balance to itself. - selfDestructContract := []byte{ - byte(vm.PUSH1), 2, // PUSH1 2 - byte(vm.PUSH1), 10, // PUSH1 12 - byte(vm.PUSH0), // PUSH0 - byte(vm.CODECOPY), // Codecopy ( to offset 0, code@offset: 10, length: 2) - - byte(vm.PUSH1), 22, - byte(vm.PUSH0), - byte(vm.RETURN), // RETURN( memory[0:2] ) - - // Deployed code - byte(vm.ADDRESS), - byte(vm.SELFDESTRUCT), - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - if i == 0 { - // Create self-destruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - { - // Check self-destructed contract in the witness. - // The way 6780 is implemented today, it always SubBalance from the self-destructed contract, and AddBalance - // to the beneficiary. In this case both addresses are the same, thus this might be optimizable from a gas - // perspective. But until that happens, we need to honor this "balance reading" adding it to the witness. - - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatal("the pre-state balance before self-destruct must be 42") - } - - // Note that the SubBalance+AddBalance net effect is a 0 change, so NewValue - // must be nil. - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be empty") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in the same tx as the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - - // The goal of this test is to test SELFDESTRUCT that happens while executing - // the init code of a contract creation, that occurs in **the same** transaction. - // The balance is sent to itself. - t.Logf("Contract: %v", contract.String()) - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - if balanceStateDiff.CurrentValue != nil { - t.Fatal("the pre-state balance before must be nil, since the contract didn't exist") - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount checks the -// content of the witness if a selfdestruct occurs in the same tx as the one that created it, -// it, but the beneficiary is the selfdestructed account. The difference with the test above, -// is that the created account is prefunded and so the final value should be 0. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount(t *testing.T) { - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - // Prefund the account, at an address that the contract will be deployed at, - // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the current version of the spec requires. - // TODO(gballet) revisit after the spec has been modified. - gspec.Alloc[contract] = types.Account{ - Balance: big.NewInt(100), - } - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - expected, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000064") - if balanceStateDiff.CurrentValue == nil || !bytes.Equal(balanceStateDiff.CurrentValue[:], expected) { - t.Fatalf("incorrect prestate balance: %x != %x", *balanceStateDiff.CurrentValue, expected) - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - expected = make([]byte, 32) - if balanceStateDiff.NewValue == nil { - t.Fatal("incorrect nil poststate balance") - } - if !bytes.Equal(balanceStateDiff.NewValue[:], expected[:]) { - t.Fatalf("incorrect poststate balance: %x != %x", *balanceStateDiff.NewValue, expected[:]) - } - } -} diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 7163083e96..06b83137ff 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -339,11 +339,11 @@ func (c *ecrecover) Run(input []byte) ([]byte, error) { } // We must make sure not to modify the 'input', so placing the 'v' along with // the signature needs to be done on a new allocation - sig := make([]byte, 65) - copy(sig, input[64:128]) + var sig [65]byte + copy(sig[:], input[64:128]) sig[64] = v // v needs to be at the end for libsecp256k1 - pubKey, err := crypto.Ecrecover(input[:32], sig) + pubKey, err := crypto.Ecrecover(input[:32], sig[:]) // make sure the public key is a valid one if err != nil { return nil, nil @@ -631,7 +631,6 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 { if expLen > 32 { expHead.SetBytes(getData(input, baseLen, 32)) } else { - // TODO: Check that if expLen < baseLen, then getData will return an empty slice expHead.SetBytes(getData(input, baseLen, expLen)) } } diff --git a/core/vm/eips.go b/core/vm/eips.go index 18bdbfdf97..37335c1cf3 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -44,6 +44,7 @@ var activators = map[int]func(*JumpTable){ 4762: enable4762, 7702: enable7702, 7939: enable7939, + 8024: enable8024, } // EnableEIP enables the given EIP on the config. @@ -347,6 +348,28 @@ func enable6780(jt *JumpTable) { } } +// enable8024 applies EIP-8024 (DUPN, SWAPN, EXCHANGE) +func enable8024(jt *JumpTable) { + jt[DUPN] = &operation{ + execute: opDupN, + constantGas: GasFastestStep, + minStack: minStack(1, 0), + maxStack: maxStack(0, 1), + } + jt[SWAPN] = &operation{ + execute: opSwapN, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } + jt[EXCHANGE] = &operation{ + execute: opExchange, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } +} + func opExtCodeCopyEIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( stack = scope.Stack diff --git a/core/vm/evm.go b/core/vm/evm.go index b49f0ac287..4694a32b91 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -79,9 +79,8 @@ type BlockContext struct { type TxContext struct { // Message information Origin common.Address // Provides information for ORIGIN - GasPrice *big.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) + GasPrice *uint256.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) BlobHashes []common.Hash // Provides information for BLOBHASH - BlobFeeCap *big.Int // Is used to zero the blobbasefee if NoBaseFee is set AccessEvents *state.AccessEvents // Capture all state accesses for this tx } @@ -133,9 +132,6 @@ type EVM struct { // jumpDests stores results of JUMPDEST analysis. jumpDests JumpDestCache - hasher crypto.KeccakState // Keccak256 hasher instance shared across opcodes - hasherBuf common.Hash // Keccak256 hasher result array shared across opcodes - readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse } @@ -152,7 +148,6 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon chainConfig: chainConfig, chainRules: chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time, blockCtx.ArbOSVersion), jumpDests: newMapJumpDests(), - hasher: crypto.NewKeccakState(), } evm.ProcessingHook = DefaultTxProcessor{evm: evm} evm.precompiles = activePrecompiledContracts(evm.chainRules) @@ -223,7 +218,7 @@ func (evm *EVM) SetJumpDestCache(jumpDests JumpDestCache) { // This is not threadsafe and should only be done very cautiously. func (evm *EVM) SetTxContext(txCtx TxContext) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + txCtx.AccessEvents = state.NewAccessEvents() } evm.TxContext = txCtx } @@ -703,7 +698,7 @@ func (evm *EVM) Create(caller common.Address, code []byte, gas uint64, value *ui // The different between Create2 with Create is Create2 uses keccak256(0xff ++ msg.sender ++ salt ++ keccak256(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller common.Address, code []byte, gas uint64, endowment *uint256.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, usedMultiGas multigas.MultiGas, err error) { - inithash := crypto.HashData(evm.hasher, code) + inithash := crypto.Keccak256Hash(code) contractAddr = crypto.CreateAddress2(caller, salt.Bytes32(), inithash[:]) return evm.create(caller, code, gas, endowment, contractAddr, CREATE2) } diff --git a/core/vm/evm_arbitrum.go b/core/vm/evm_arbitrum.go index 87c774e6e6..b489831977 100644 --- a/core/vm/evm_arbitrum.go +++ b/core/vm/evm_arbitrum.go @@ -105,7 +105,7 @@ func (p DefaultTxProcessor) L1BlockHash(blockCtx BlockContext, l1BlocKNumber uin } func (p DefaultTxProcessor) GasPriceOp(evm *EVM) *big.Int { - return evm.GasPrice + return evm.GasPrice.ToBig() } func (p DefaultTxProcessor) FillReceiptInfo(*types.Receipt) {} diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 189963634b..614f4bd0af 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -112,6 +112,9 @@ var ( ) func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } var ( y, x = stack.Back(1), stack.Back(0) current, original = evm.StateDB.GetStateAndCommittedState(contract.Address(), x.Bytes32()) @@ -197,6 +200,9 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi // (2.2.2.1.) If original value is 0, add SSTORE_SET_GAS - SLOAD_GAS to refund counter. // (2.2.2.2.) Otherwise, add SSTORE_RESET_GAS - SLOAD_GAS gas to refund counter. func gasSStoreEIP2200(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } // If we fail the minimum gas availability invariant, fail (0) if contract.Gas <= params.SstoreSentryGasEIP2200 { return multigas.ZeroGas(), errors.New("not enough gas for reentrancy sentry") @@ -424,6 +430,10 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize address = common.Address(stack.Back(1).Bytes20()) ) + if evm.readOnly && transfersValue { + return multigas.ZeroGas(), ErrWriteProtection + } + // Storage slot writes (zero → nonzero) considered as storage growth. // See rationale in: https://github.com/OffchainLabs/nitro/blob/master/docs/decisions/0002-multi-dimensional-gas-metering.md if evm.chainRules.IsEIP158 { @@ -536,6 +546,9 @@ func gasStaticCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memo } func gasSelfdestruct(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } multiGas := multigas.ZeroGas() // EIP150 homestead gas reprice fork: if evm.chainRules.IsEIP150 { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 86cbf9697c..20452526fa 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -237,14 +238,12 @@ func opKeccak256(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.peek() data := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) - evm.hasher.Reset() - evm.hasher.Write(data) - evm.hasher.Read(evm.hasherBuf[:]) + hash := crypto.Keccak256Hash(data) if evm.Config.EnablePreimageRecording { - evm.StateDB.AddPreimage(evm.hasherBuf, data) + evm.StateDB.AddPreimage(hash, data) } - size.SetBytes(evm.hasherBuf[:]) + size.SetBytes(hash[:]) return nil, nil } @@ -923,21 +922,34 @@ func opSelfdestruct(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { if evm.readOnly { return nil, ErrWriteProtection } - beneficiary := scope.Stack.pop() + var ( + this = scope.Contract.Address() + balance = evm.StateDB.GetBalance(this) + top = scope.Stack.pop() + beneficiary = common.Address(top.Bytes20()) + ) + evm.StateDB.TouchAddress(&filter.FilteredAddressRecord{ - Address: beneficiary.Bytes20(), + Address: beneficiary, FilterReason: filter.FilterReason{Reason: filter.ReasonSelfdestructBeneficiary, EventRuleMatch: nil}, }) - balance := evm.StateDB.GetBalance(scope.Contract.Address()) - evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) - evm.StateDB.SelfDestruct(scope.Contract.Address()) - if beneficiary.Bytes20() == scope.Contract.Address() { + + // The funds are burned immediately if the beneficiary is the caller itself, + // in this case, the beneficiary's balance is not increased. + if this != beneficiary { + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + // Clear any leftover funds for the account being destructed. + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.SelfDestruct(this) + if this == beneficiary { // Arbitrum: calling selfdestruct(this) burns the balance evm.StateDB.ExpectBalanceBurn(balance.ToBig()) } + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnEnter != nil { - tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig()) } if tracer.OnExit != nil { tracer.OnExit(evm.depth, []byte{}, 0, nil, false) @@ -950,33 +962,50 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro if evm.readOnly { return nil, ErrWriteProtection } + var ( + this = scope.Contract.Address() + balance = evm.StateDB.GetBalance(this) + top = scope.Stack.pop() + beneficiary = common.Address(top.Bytes20()) + newContract = evm.StateDB.IsNewContract(this) + ) // Arbitrum: revert if acting account is a Stylus program - actingAddress := scope.Contract.Address() - if code := evm.StateDB.GetCode(actingAddress); state.IsStylusComponentPrefix(code, evm.chainRules.ArbOSVersion) { + if code := evm.StateDB.GetCode(this); state.IsStylusComponentPrefix(code, evm.chainRules.ArbOSVersion) { return nil, ErrExecutionReverted } - beneficiary := scope.Stack.pop() evm.StateDB.TouchAddress(&filter.FilteredAddressRecord{ - Address: beneficiary.Bytes20(), + Address: beneficiary, FilterReason: filter.FilterReason{Reason: filter.ReasonSelfdestructBeneficiary, EventRuleMatch: nil}, }) - balance := evm.StateDB.GetBalance(scope.Contract.Address()) - evm.StateDB.SubBalance(scope.Contract.Address(), balance, tracing.BalanceDecreaseSelfdestruct) - evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct) - evm.StateDB.SelfDestruct6780(scope.Contract.Address()) - if beneficiary.Bytes20() == scope.Contract.Address() { + + // Contract is new and will actually be deleted. + if newContract { + if this != beneficiary { // Skip no-op transfer when self-destructing to self. + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.SelfDestruct(this) + } + + // Contract already exists, only do transfer if beneficiary is not self. + if !newContract && this != beneficiary { + evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct) + evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct) + } + + if this == beneficiary { // SelfDestruct6780 only destructs the contract if selfdestructing in the same transaction as contract creation // So we only account for the balance burn if the contract is actually destructed by checking if the balance is zero. - if evm.StateDB.GetBalance(scope.Contract.Address()).Sign() == 0 { + if evm.StateDB.GetBalance(this).Sign() == 0 { // Arbitrum: calling selfdestruct(this) burns the balance evm.StateDB.ExpectBalanceBurn(balance.ToBig()) } } if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnEnter != nil { - tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig()) + tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig()) } if tracer.OnExit != nil { tracer.OnExit(evm.depth, []byte{}, 0, nil, false) @@ -985,6 +1014,115 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro return nil, errStopToken } +func decodeSingle(x byte) int { + if x <= 90 { + return int(x) + 17 + } + return int(x) - 20 +} + +func decodePair(x byte) (int, int) { + var k int + if x <= 79 { + k = int(x) + } else { + k = int(x) - 48 + } + q, r := k/16, k%16 + if q < r { + return q + 1, r + 1 + } + return r + 1, 29 - q +} + +func opDupN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // DUPN duplicates the n'th stack item, so the stack must contain at least n elements. + if scope.Stack.len() < n { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n} + } + + //The n‘th stack item is duplicated at the top of the stack. + scope.Stack.push(scope.Stack.Back(n - 1)) + *pc += 1 + return nil, nil +} + +func opSwapN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // SWAPN operates on the top and n+1 stack items, so the stack must contain at least n+1 elements. + if scope.Stack.len() < n+1 { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n + 1} + } + + // The (n+1)‘th stack item is swapped with the top of the stack. + indexTop := scope.Stack.len() - 1 + indexN := scope.Stack.len() - 1 - n + scope.Stack.data[indexTop], scope.Stack.data[indexN] = scope.Stack.data[indexN], scope.Stack.data[indexTop] + *pc += 1 + return nil, nil +} + +func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded both to preserve compatibility with existing opcodes + // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–79, 128–255). + if x > 79 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n, m := decodePair(x) + need := max(n, m) + 1 + + // EXCHANGE operates on the (n+1)'th and (m+1)'th stack items, + // so the stack must contain at least max(n, m)+1 elements. + if scope.Stack.len() < need { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: need} + } + + // The (n+1)‘th stack item is swapped with the (m+1)‘th stack item. + indexN := scope.Stack.len() - 1 - n + indexM := scope.Stack.len() - 1 - m + scope.Stack.data[indexN], scope.Stack.data[indexM] = scope.Stack.data[indexM], scope.Stack.data[indexN] + *pc += 1 + return nil, nil +} + // following functions are used by the instruction jump table // make log instruction function diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index bf027ce110..d03c6ba0a9 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -19,6 +19,7 @@ package vm import ( "bytes" "encoding/json" + "errors" "fmt" "math/big" "os" @@ -1081,3 +1082,222 @@ func TestOpSelfdestruct6780NonStylusCodeProceeds(t *testing.T) { }) } } + +func TestEIP8024_Execution(t *testing.T) { + evm := NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + + tests := []struct { + name string + codeHex string + wantErr error + wantOpcode OpCode + wantVals []uint64 + }{ + { + name: "DUPN", + codeHex: "60016000808080808080808080808080808080e600", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "DUPN_MISSING_IMMEDIATE", + codeHex: "60016000808080808080808080808080808080e6", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "SWAPN", + codeHex: "600160008080808080808080808080808080806002e700", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, + }, + }, + { + name: "SWAPN_MISSING_IMMEDIATE", + codeHex: "600160008080808080808080808080808080806002e7", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, + }, + }, + { + name: "EXCHANGE", + codeHex: "600060016002e801", + wantVals: []uint64{2, 0, 1}, + }, + { + name: "EXCHANGE_MISSING_IMMEDIATE", + codeHex: "600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060016002e8", + wantVals: []uint64{ + 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "INVALID_SWAPN_LOW", + codeHex: "e75b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, + }, + { + name: "JUMP over INVALID_DUPN", + codeHex: "600456e65b", + wantErr: nil, + }, + { + name: "UNDERFLOW_DUPN_1", + codeHex: "6000808080808080808080808080808080e600", + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, + }, + // Additional test cases + { + name: "INVALID_DUPN_LOW", + codeHex: "e65b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, + }, + { + name: "INVALID_EXCHANGE_LOW", + codeHex: "e850", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "INVALID_DUPN_HIGH", + codeHex: "e67f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, + }, + { + name: "INVALID_SWAPN_HIGH", + codeHex: "e77f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, + }, + { + name: "INVALID_EXCHANGE_HIGH", + codeHex: "e87f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "UNDERFLOW_DUPN_2", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, + }, + { + name: "UNDERFLOW_SWAPN", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) + wantErr: &ErrStackUnderflow{}, + wantOpcode: SWAPN, + }, + { + name: "UNDERFLOW_EXCHANGE", + codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 + wantErr: &ErrStackUnderflow{}, + wantOpcode: EXCHANGE, + }, + { + name: "PC_INCREMENT", + codeHex: "600060006000e80115", + wantVals: []uint64{1, 0, 0}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code := common.FromHex(tc.codeHex) + stack := newstack() + pc := uint64(0) + scope := &ScopeContext{Stack: stack, Contract: &Contract{Code: code}} + var err error + var errOp OpCode + for pc < uint64(len(code)) && err == nil { + op := code[pc] + switch OpCode(op) { + case STOP: + return + case PUSH1: + _, err = opPush1(&pc, evm, scope) + case DUP1: + dup1 := makeDup(1) + _, err = dup1(&pc, evm, scope) + case JUMP: + _, err = opJump(&pc, evm, scope) + case JUMPDEST: + _, err = opJumpdest(&pc, evm, scope) + case ISZERO: + _, err = opIszero(&pc, evm, scope) + case PUSH0: + _, err = opPush0(&pc, evm, scope) + case DUPN: + _, err = opDupN(&pc, evm, scope) + case SWAPN: + _, err = opSwapN(&pc, evm, scope) + case EXCHANGE: + _, err = opExchange(&pc, evm, scope) + default: + t.Fatalf("unexpected opcode %s at pc=%d", OpCode(op), pc) + } + if err != nil { + errOp = OpCode(op) + } + pc++ + } + if tc.wantErr != nil { + // Fail because we wanted an error, but didn't get one. + if err == nil { + t.Fatalf("expected error, got nil") + } + // Fail if the wrong opcode threw an error. + if errOp != tc.wantOpcode { + t.Fatalf("expected error from opcode %s, got %s", tc.wantOpcode, errOp) + } + // Fail if we don't get the error we expect. + switch tc.wantErr.(type) { + case *ErrInvalidOpCode: + var want *ErrInvalidOpCode + if !errors.As(err, &want) { + t.Fatalf("expected ErrInvalidOpCode, got %v", err) + } + case *ErrStackUnderflow: + var want *ErrStackUnderflow + if !errors.As(err, &want) { + t.Fatalf("expected ErrStackUnderflow, got %v", err) + } + default: + t.Fatalf("unsupported wantErr type %T", tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := make([]uint64, 0, stack.len()) + for i := stack.len() - 1; i >= 0; i-- { + got = append(got, stack.data[i].Uint64()) + } + if len(got) != len(tc.wantVals) { + t.Fatalf("stack len=%d; want %d", len(got), len(tc.wantVals)) + } + for i := range got { + if got[i] != tc.wantVals[i] { + t.Fatalf("[%s] stack[%d]=%d; want %d\nstack=%v", + tc.name, i, got[i], tc.wantVals[i], got) + } + } + }) + } +} diff --git a/core/vm/interface.go b/core/vm/interface.go index f3a38a4e3c..c797a1320d 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -95,20 +94,18 @@ type StateDB interface { GetTransientState(addr common.Address, key common.Hash) common.Hash SetTransientState(addr common.Address, key, value common.Hash) - SelfDestruct(common.Address) uint256.Int + SelfDestruct(common.Address) HasSelfDestructed(common.Address) bool GetSelfDestructs() []common.Address - // SelfDestruct6780 is post-EIP6780 selfdestruct, which means that it's a - // send-all-to-beneficiary, unless the contract was created in this same - // transaction, in which case it will be destructed. - // This method returns the prior balance, along with a boolean which is - // true iff the object was indeed destructed. - SelfDestruct6780(common.Address) (uint256.Int, bool) - // Exist reports whether the given account exists in state. // Notably this also returns true for self-destructed accounts within the current transaction. Exist(common.Address) bool + + // IsNewContract reports whether the contract at the given address was deployed + // during the current transaction. + IsNewContract(addr common.Address) bool + // Empty returns whether the given account is empty. Empty // is defined according to EIP161 (balance = nonce = code = 0). Empty(common.Address) bool @@ -124,9 +121,6 @@ type StateDB interface { // GetAccessList returns the access list GetAccessList() (addresses map[common.Address]int, slots []map[common.Hash]struct{}) - // PointCache returns the point cache used in computations - PointCache() *utils.PointCache - Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) RevertToSnapshot(int) diff --git a/core/vm/memory.go b/core/vm/memory.go index 5e11e83748..54bc2b2849 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -44,6 +44,7 @@ func (m *Memory) Free() { // To reduce peak allocation, return only smaller memory instances to the pool. const maxBufferSize = 16 << 10 if cap(m.store) <= maxBufferSize { + clear(m.store) m.store = m.store[:0] m.lastGasCost = 0 memoryPool.Put(m) @@ -76,10 +77,14 @@ func (m *Memory) Set32(offset uint64, val *uint256.Int) { val.PutUint256(m.store[offset:]) } -// Resize resizes the memory to size +// Resize grows the memory to the requested size. func (m *Memory) Resize(size uint64) { - if uint64(m.Len()) < size { - m.store = append(m.store, make([]byte, size-uint64(m.Len()))...) + if uint64(len(m.store)) < size { + if uint64(cap(m.store)) >= size { + m.store = m.store[:size] + } else { + m.store = append(m.store, make([]byte, size-uint64(len(m.store)))...) + } } } diff --git a/core/vm/memory_test.go b/core/vm/memory_test.go index 41389b729a..3890d18cb5 100644 --- a/core/vm/memory_test.go +++ b/core/vm/memory_test.go @@ -83,3 +83,10 @@ func TestMemoryCopy(t *testing.T) { } } } + +func BenchmarkResize(b *testing.B) { + memory := NewMemory() + for i := range b.N { + memory.Resize(uint64(i)) + } +} diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index ca42d8148c..63455cba27 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -28,6 +28,9 @@ import ( func makeGasSStoreFunc(clearingRefund uint64) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } // If we fail the minimum gas availability invariant, fail (0) if contract.Gas <= params.SstoreSentryGasEIP2200 { return multigas.ZeroGas(), errors.New("not enough gas for reentrancy sentry") @@ -257,12 +260,21 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { multiGas = multigas.ZeroGas() address = common.Address(stack.peek().Bytes20()) ) + if evm.readOnly { + return multigas.ZeroGas(), ErrWriteProtection + } if !evm.StateDB.AddressInAccessList(address) { // If the caller cannot afford the cost, this change will be rolled back evm.StateDB.AddAddressToAccessList(address) // Cold account access considered as storage access read. // See rationale in: https://github.com/OffchainLabs/nitro/blob/master/docs/decisions/0002-multi-dimensional-gas-metering.md multiGas = multiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, params.ColdAccountAccessCostEIP2929) + + // Terminate the gas measurement if the leftover gas is not sufficient, + // it can effectively prevent accessing the states in the following steps + if contract.Gas < multiGas.SingleGas() { + return multigas.ZeroGas(), ErrOutOfGas + } } // if empty and transfers value if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 { @@ -279,12 +291,24 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { } var ( - gasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) + innerGasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCall) gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCall) gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCode) ) +func gasCallEIP7702(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { + // Return early if this call attempts to transfer value in a static context. + // Although it's checked in `gasCall`, EIP-7702 loads the target's code before + // to determine if it is resolving a delegation. This could incorrectly record + // the target in the block access list (BAL) if the call later fails. + transfersValue := !stack.Back(2).IsZero() + if evm.readOnly && transfersValue { + return multigas.ZeroGas(), ErrWriteProtection + } + return innerGasCallEIP7702(evm, contract, stack, mem, memorySize) +} + func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (multigas.MultiGas, error) { var ( diff --git a/core/vm/operations_gas_test.go b/core/vm/operations_gas_test.go index bea744b222..9672d0b919 100644 --- a/core/vm/operations_gas_test.go +++ b/core/vm/operations_gas_test.go @@ -420,7 +420,7 @@ func TestGasSStore4762(t *testing.T) { mem := NewMemory() // Setup access list and stack - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() accessList.AddAccount(caller, false, 0) evm.AccessEvents = accessList @@ -488,7 +488,7 @@ func testGasSLoad(t *testing.T, gasFunc gasFunc, slotKeyProviders ...func(contra mem := NewMemory() // Setup access list and stack - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() accessList.AddAccount(caller, false, 0) evm.AccessEvents = accessList @@ -589,7 +589,7 @@ func testGasCallFuncFuncWithCases(t *testing.T, config *params.ChainConfig, gasC // Mock AccessEvents for EIP4762 if tc.isEIP4762 { - accessEvents := state.NewAccessEvents(stateDb.PointCache()) + accessEvents := state.NewAccessEvents() evm.AccessEvents = accessEvents } @@ -1263,8 +1263,8 @@ func TestGasSelfdestructEIP4762(t *testing.T) { mem := NewMemory() // Setup access list - accessList := state.NewAccessEvents(evm.StateDB.PointCache()) - accessListForExpected := state.NewAccessEvents(evm.StateDB.PointCache()) + accessList := state.NewAccessEvents() + accessListForExpected := state.NewAccessEvents() evm.AccessEvents = accessList stateDb.CreateAccount(beneficiaryAddr) @@ -1437,7 +1437,7 @@ func TestGasCodeCopyEIP4762(t *testing.T) { // EVM + access events statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, statedb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0x1234"), new(uint256.Int), 100000, nil) contract.Code = make([]byte, c.codeLen) @@ -1464,7 +1464,7 @@ func TestGasCodeCopyEIP4762(t *testing.T) { if c.expectChunks { _, copyOff, nonPadded := getDataAndAdjustedBounds(contract.Code, c.codeOffset, c.length) single := expectedMultiGas.SingleGas() - aeExp := state.NewAccessEvents(evm.StateDB.PointCache()) + aeExp := state.NewAccessEvents() _, wanted := aeExp.CodeChunksRangeGas(contract.Address(), copyOff, nonPadded, uint64(len(contract.Code)), false, contract.Gas-single) expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, wanted) } @@ -1519,7 +1519,7 @@ func TestGasExtCodeCopyEIP4762(t *testing.T) { // EVM + access events stateDb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, stateDb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0xBEEF"), new(uint256.Int), 100000, nil) @@ -1545,7 +1545,7 @@ func TestGasExtCodeCopyEIP4762(t *testing.T) { expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindComputation, params.WarmStorageReadCostEIP2929) } else { singleGas := expectedMultiGas.SingleGas() - accessEventsForExpected := state.NewAccessEvents(evm.StateDB.PointCache()) + accessEventsForExpected := state.NewAccessEvents() wgas := accessEventsForExpected.BasicDataGas(c.addr, false, contract.Gas-singleGas, true) expectedMultiGas = expectedMultiGas.SaturatingIncrement(multigas.ResourceKindStorageAccessRead, wgas) } @@ -1724,7 +1724,7 @@ func TestGasBalanceExtCodeSizeExtCodeHash4762(t *testing.T) { // EVM + AccessEvents stateDb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm := NewEVM(BlockContext{BlockNumber: big.NewInt(0)}, stateDb, params.TestChainConfig, Config{}) - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() contract := NewContract(common.Address{}, common.HexToAddress("0xBEEF"), new(uint256.Int), 100000, nil) @@ -1744,7 +1744,7 @@ func TestGasBalanceExtCodeSizeExtCodeHash4762(t *testing.T) { } // Reset AccessEvents so the call sees the same state - evm.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + evm.AccessEvents = state.NewAccessEvents() multiGas, err := tt.gasFn(evm, contract, stack, mem, 0) if err != nil { diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index 6865f72a3f..2632cce6d7 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -19,6 +19,7 @@ package runtime import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" ) func NewEnv(cfg *Config) *vm.EVM { @@ -27,9 +28,8 @@ func NewEnv(cfg *Config) *vm.EVM { } txContext := vm.TxContext{ Origin: cfg.Origin, - GasPrice: cfg.GasPrice, + GasPrice: uint256.MustFromBig(cfg.GasPrice), BlobHashes: cfg.BlobHashes, - BlobFeeCap: cfg.BlobFeeCap, } blockContext := vm.BlockContext{ CanTransfer: core.CanTransfer, diff --git a/crypto/ecies/ecies.go b/crypto/ecies/ecies.go index 9a892781f4..378d764a19 100644 --- a/crypto/ecies/ecies.go +++ b/crypto/ecies/ecies.go @@ -124,6 +124,9 @@ func (prv *PrivateKey) GenerateShared(pub *PublicKey, skLen, macLen int) (sk []b if prv.PublicKey.Curve != pub.Curve { return nil, ErrInvalidCurve } + if pub.X == nil || pub.Y == nil || !pub.Curve.IsOnCurve(pub.X, pub.Y) { + return nil, ErrInvalidPublicKey + } if skLen+macLen > MaxSharedKeyLength(pub) { return nil, ErrSharedKeyTooBig } diff --git a/crypto/keccak/LICENSE b/crypto/keccak/LICENSE new file mode 100644 index 0000000000..2a7cf70da6 --- /dev/null +++ b/crypto/keccak/LICENSE @@ -0,0 +1,27 @@ +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/keccak/README.md b/crypto/keccak/README.md new file mode 100644 index 0000000000..295a5b958c --- /dev/null +++ b/crypto/keccak/README.md @@ -0,0 +1,6 @@ +This is a vendored and modified copy of golang.org/x/crypto/sha3, with an assembly +implementation of keccak256. We wish to retain the assembly implementation, +which was removed in v0.44.0. + +Ethereum uses a 'legacy' variant of Keccak, which was defined before it became SHA3. As +such, we cannot use the standard library crypto/sha3 package. diff --git a/crypto/keccak/hashes.go b/crypto/keccak/hashes.go new file mode 100644 index 0000000000..c78c5fe992 --- /dev/null +++ b/crypto/keccak/hashes.go @@ -0,0 +1,44 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// This file provides functions for creating instances of the SHA-3 +// and SHAKE hash functions, as well as utility functions for hashing +// bytes. + +import ( + "hash" +) + +const ( + dsbyteSHA3 = 0b00000110 + dsbyteKeccak = 0b00000001 + dsbyteShake = 0b00011111 + dsbyteCShake = 0b00000100 + + // rateK[c] is the rate in bytes for Keccak[c] where c is the capacity in + // bits. Given the sponge size is 1600 bits, the rate is 1600 - c bits. + rateK256 = (1600 - 256) / 8 + rateK448 = (1600 - 448) / 8 + rateK512 = (1600 - 512) / 8 + rateK768 = (1600 - 768) / 8 + rateK1024 = (1600 - 1024) / 8 +) + +// NewLegacyKeccak256 creates a new Keccak-256 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New256 instead. +func NewLegacyKeccak256() hash.Hash { + return &state{rate: rateK512, outputLen: 32, dsbyte: dsbyteKeccak} +} + +// NewLegacyKeccak512 creates a new Keccak-512 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New512 instead. +func NewLegacyKeccak512() hash.Hash { + return &state{rate: rateK1024, outputLen: 64, dsbyte: dsbyteKeccak} +} diff --git a/crypto/keccak/keccakf.go b/crypto/keccak/keccakf.go new file mode 100644 index 0000000000..82694fa4a3 --- /dev/null +++ b/crypto/keccak/keccakf.go @@ -0,0 +1,414 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !amd64 || purego || !gc + +package keccak + +import "math/bits" + +// rc stores the round constants for use in the ι step. +var rc = [24]uint64{ + 0x0000000000000001, + 0x0000000000008082, + 0x800000000000808A, + 0x8000000080008000, + 0x000000000000808B, + 0x0000000080000001, + 0x8000000080008081, + 0x8000000000008009, + 0x000000000000008A, + 0x0000000000000088, + 0x0000000080008009, + 0x000000008000000A, + 0x000000008000808B, + 0x800000000000008B, + 0x8000000000008089, + 0x8000000000008003, + 0x8000000000008002, + 0x8000000000000080, + 0x000000000000800A, + 0x800000008000000A, + 0x8000000080008081, + 0x8000000000008080, + 0x0000000080000001, + 0x8000000080008008, +} + +// keccakF1600 applies the Keccak permutation to a 1600b-wide +// state represented as a slice of 25 uint64s. +func keccakF1600(a *[25]uint64) { + // Implementation translated from Keccak-inplace.c + // in the keccak reference code. + var t, bc0, bc1, bc2, bc3, bc4, d0, d1, d2, d3, d4 uint64 + + for i := 0; i < 24; i += 4 { + // Combines the 5 steps in each round into 2 steps. + // Unrolls 4 rounds per loop and spreads some steps across rounds. + + // Round 1 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[6] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[12] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[18] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[24] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i] + a[6] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[16] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[22] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[3] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[10] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[1] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[7] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[19] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[20] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[11] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[23] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[4] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[5] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[2] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[8] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[14] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[15] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + // Round 2 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[16] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[7] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[23] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[14] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+1] + a[16] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[11] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[2] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[18] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[20] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[6] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[22] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[4] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[15] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[1] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[8] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[24] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[10] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[12] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[3] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[19] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[5] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + // Round 3 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[11] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[22] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[8] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[19] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+2] + a[11] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[1] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[12] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[23] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[15] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[16] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[2] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[24] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[5] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[6] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[3] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[14] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[20] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[7] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[18] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[4] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[10] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + // Round 4 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[1] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[2] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[3] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[4] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+3] + a[1] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[6] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[7] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[8] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[5] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[11] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[12] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[14] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[10] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[16] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[18] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[19] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[15] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[22] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[23] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[24] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[20] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + } +} diff --git a/crypto/keccak/keccakf_amd64.go b/crypto/keccak/keccakf_amd64.go new file mode 100644 index 0000000000..cb6eca44c3 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego && gc + +package keccak + +// This function is implemented in keccakf_amd64.s. + +//go:noescape + +func keccakF1600(a *[25]uint64) diff --git a/crypto/keccak/keccakf_amd64.s b/crypto/keccak/keccakf_amd64.s new file mode 100644 index 0000000000..99e2f16e97 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.s @@ -0,0 +1,5419 @@ +// Code generated by command: go run keccakf_amd64_asm.go -out ../keccakf_amd64.s -pkg sha3. DO NOT EDIT. + +//go:build amd64 && !purego && gc + +// func keccakF1600(a *[25]uint64) +TEXT ·keccakF1600(SB), $200-8 + MOVQ a+0(FP), DI + + // Convert the user state into an internal state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + + // Execute the KeccakF permutation + MOVQ (DI), SI + MOVQ 8(DI), BP + MOVQ 32(DI), R15 + XORQ 40(DI), SI + XORQ 48(DI), BP + XORQ 72(DI), R15 + XORQ 80(DI), SI + XORQ 88(DI), BP + XORQ 112(DI), R15 + XORQ 120(DI), SI + XORQ 128(DI), BP + XORQ 152(DI), R15 + XORQ 160(DI), SI + XORQ 168(DI), BP + MOVQ 176(DI), DX + MOVQ 184(DI), R8 + XORQ 192(DI), R15 + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000008082, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000808a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008000, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000008a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000088, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000008b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008089, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008003, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008002, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000000080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000800a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008008, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + NOP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + NOP + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + NOP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + NOP + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + NOP + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + NOP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + NOP + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + NOP + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + NOP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + NOP + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + NOP + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + NOP + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + NOP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Revert the internal state to the user state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + RET diff --git a/crypto/keccak/sha3.go b/crypto/keccak/sha3.go new file mode 100644 index 0000000000..a554323244 --- /dev/null +++ b/crypto/keccak/sha3.go @@ -0,0 +1,244 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +import ( + "crypto/subtle" + "encoding/binary" + "errors" + "unsafe" + + "golang.org/x/sys/cpu" +) + +// spongeDirection indicates the direction bytes are flowing through the sponge. +type spongeDirection int + +const ( + // spongeAbsorbing indicates that the sponge is absorbing input. + spongeAbsorbing spongeDirection = iota + // spongeSqueezing indicates that the sponge is being squeezed. + spongeSqueezing +) + +type state struct { + a [1600 / 8]byte // main state of the hash + + // a[n:rate] is the buffer. If absorbing, it's the remaining space to XOR + // into before running the permutation. If squeezing, it's the remaining + // output to produce before running the permutation. + n, rate int + + // dsbyte contains the "domain separation" bits and the first bit of + // the padding. Sections 6.1 and 6.2 of [1] separate the outputs of the + // SHA-3 and SHAKE functions by appending bitstrings to the message. + // Using a little-endian bit-ordering convention, these are "01" for SHA-3 + // and "1111" for SHAKE, or 00000010b and 00001111b, respectively. Then the + // padding rule from section 5.1 is applied to pad the message to a multiple + // of the rate, which involves adding a "1" bit, zero or more "0" bits, and + // a final "1" bit. We merge the first "1" bit from the padding into dsbyte, + // giving 00000110b (0x06) and 00011111b (0x1f). + // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf + // "Draft FIPS 202: SHA-3 Standard: Permutation-Based Hash and + // Extendable-Output Functions (May 2014)" + dsbyte byte + + outputLen int // the default output size in bytes + state spongeDirection // whether the sponge is absorbing or squeezing +} + +// BlockSize returns the rate of sponge underlying this hash function. +func (d *state) BlockSize() int { return d.rate } + +// Size returns the output size of the hash function in bytes. +func (d *state) Size() int { return d.outputLen } + +// Reset clears the internal state by zeroing the sponge state and +// the buffer indexes, and setting Sponge.state to absorbing. +func (d *state) Reset() { + // Zero the permutation's state. + for i := range d.a { + d.a[i] = 0 + } + d.state = spongeAbsorbing + d.n = 0 +} + +func (d *state) clone() *state { + ret := *d + return &ret +} + +// permute applies the KeccakF-1600 permutation. +func (d *state) permute() { + var a *[25]uint64 + if cpu.IsBigEndian { + a = new([25]uint64) + for i := range a { + a[i] = binary.LittleEndian.Uint64(d.a[i*8:]) + } + } else { + a = (*[25]uint64)(unsafe.Pointer(&d.a)) + } + + keccakF1600(a) + d.n = 0 + + if cpu.IsBigEndian { + for i := range a { + binary.LittleEndian.PutUint64(d.a[i*8:], a[i]) + } + } +} + +// pads appends the domain separation bits in dsbyte, applies +// the multi-bitrate 10..1 padding rule, and permutes the state. +func (d *state) padAndPermute() { + // Pad with this instance's domain-separator bits. We know that there's + // at least one byte of space in the sponge because, if it were full, + // permute would have been called to empty it. dsbyte also contains the + // first one bit for the padding. See the comment in the state struct. + d.a[d.n] ^= d.dsbyte + // This adds the final one bit for the padding. Because of the way that + // bits are numbered from the LSB upwards, the final bit is the MSB of + // the last byte. + d.a[d.rate-1] ^= 0x80 + // Apply the permutation + d.permute() + d.state = spongeSqueezing +} + +// Write absorbs more data into the hash's state. It panics if any +// output has already been read. +func (d *state) Write(p []byte) (n int, err error) { + if d.state != spongeAbsorbing { + panic("sha3: Write after Read") + } + + n = len(p) + + for len(p) > 0 { + x := subtle.XORBytes(d.a[d.n:d.rate], d.a[d.n:d.rate], p) + d.n += x + p = p[x:] + + // If the sponge is full, apply the permutation. + if d.n == d.rate { + d.permute() + } + } + + return +} + +// Read squeezes an arbitrary number of bytes from the sponge. +func (d *state) Read(out []byte) (n int, err error) { + // If we're still absorbing, pad and apply the permutation. + if d.state == spongeAbsorbing { + d.padAndPermute() + } + + n = len(out) + + // Now, do the squeezing. + for len(out) > 0 { + // Apply the permutation if we've squeezed the sponge dry. + if d.n == d.rate { + d.permute() + } + + x := copy(out, d.a[d.n:d.rate]) + d.n += x + out = out[x:] + } + + return +} + +// Sum applies padding to the hash state and then squeezes out the desired +// number of output bytes. It panics if any output has already been read. +func (d *state) Sum(in []byte) []byte { + if d.state != spongeAbsorbing { + panic("sha3: Sum after Read") + } + + // Make a copy of the original hash so that caller can keep writing + // and summing. + dup := d.clone() + hash := make([]byte, dup.outputLen, 64) // explicit cap to allow stack allocation + dup.Read(hash) + return append(in, hash...) +} + +const ( + magicSHA3 = "sha\x08" + magicShake = "sha\x09" + magicCShake = "sha\x0a" + magicKeccak = "sha\x0b" + // magic || rate || main state || n || sponge direction + marshaledSize = len(magicSHA3) + 1 + 200 + 1 + 1 +) + +func (d *state) MarshalBinary() ([]byte, error) { + return d.AppendBinary(make([]byte, 0, marshaledSize)) +} + +func (d *state) AppendBinary(b []byte) ([]byte, error) { + switch d.dsbyte { + case dsbyteSHA3: + b = append(b, magicSHA3...) + case dsbyteShake: + b = append(b, magicShake...) + case dsbyteCShake: + b = append(b, magicCShake...) + case dsbyteKeccak: + b = append(b, magicKeccak...) + default: + panic("unknown dsbyte") + } + // rate is at most 168, and n is at most rate. + b = append(b, byte(d.rate)) + b = append(b, d.a[:]...) + b = append(b, byte(d.n), byte(d.state)) + return b, nil +} + +func (d *state) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("sha3: invalid hash state") + } + + magic := string(b[:len(magicSHA3)]) + b = b[len(magicSHA3):] + switch { + case magic == magicSHA3 && d.dsbyte == dsbyteSHA3: + case magic == magicShake && d.dsbyte == dsbyteShake: + case magic == magicCShake && d.dsbyte == dsbyteCShake: + case magic == magicKeccak && d.dsbyte == dsbyteKeccak: + default: + return errors.New("sha3: invalid hash state identifier") + } + + rate := int(b[0]) + b = b[1:] + if rate != d.rate { + return errors.New("sha3: invalid hash state function") + } + + copy(d.a[:], b) + b = b[len(d.a):] + + n, state := int(b[0]), spongeDirection(b[1]) + if n > d.rate { + return errors.New("sha3: invalid hash state") + } + d.n = n + if state != spongeAbsorbing && state != spongeSqueezing { + return errors.New("sha3: invalid hash state") + } + d.state = state + + return nil +} diff --git a/crypto/keccak/sha3_test.go b/crypto/keccak/sha3_test.go new file mode 100644 index 0000000000..28a20ec72d --- /dev/null +++ b/crypto/keccak/sha3_test.go @@ -0,0 +1,210 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// Tests include all the ShortMsgKATs provided by the Keccak team at +// https://github.com/gvanas/KeccakCodePackage +// +// They only include the zero-bit case of the bitwise testvectors +// published by NIST in the draft of FIPS-202. + +import ( + "bytes" + "compress/flate" + "encoding" + "encoding/hex" + "encoding/json" + "hash" + "math/rand" + "os" + "strings" + "testing" +) + +const ( + testString = "brekeccakkeccak koax koax" + katFilename = "testdata/keccakKats.json.deflate" +) + +// testDigests contains functions returning hash.Hash instances +// with output-length equal to the KAT length for SHA-3, Keccak +// and SHAKE instances. +var testDigests = map[string]func() hash.Hash{ + "Keccak-256": NewLegacyKeccak256, + "Keccak-512": NewLegacyKeccak512, +} + +// decodeHex converts a hex-encoded string into a raw byte string. +func decodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +// structs used to marshal JSON test-cases. +type KeccakKats struct { + Kats map[string][]struct { + Digest string `json:"digest"` + Length int64 `json:"length"` + Message string `json:"message"` + + // Defined only for cSHAKE + N string `json:"N"` + S string `json:"S"` + } +} + +// TestKeccakKats tests the SHA-3 and Shake implementations against all the +// ShortMsgKATs from https://github.com/gvanas/KeccakCodePackage +// (The testvectors are stored in keccakKats.json.deflate due to their length.) +func TestKeccakKats(t *testing.T) { + // Read the KATs. + deflated, err := os.Open(katFilename) + if err != nil { + t.Errorf("error opening %s: %s", katFilename, err) + } + file := flate.NewReader(deflated) + dec := json.NewDecoder(file) + var katSet KeccakKats + err = dec.Decode(&katSet) + if err != nil { + t.Errorf("error decoding KATs: %s", err) + } + + for algo, function := range testDigests { + d := function() + for _, kat := range katSet.Kats[algo] { + d.Reset() + in, err := hex.DecodeString(kat.Message) + if err != nil { + t.Errorf("error decoding KAT: %s", err) + } + d.Write(in[:kat.Length/8]) + got := strings.ToUpper(hex.EncodeToString(d.Sum(nil))) + if got != kat.Digest { + t.Errorf("function=%s, length=%d\nmessage:\n %s\ngot:\n %s\nwanted:\n %s", + algo, kat.Length, kat.Message, got, kat.Digest) + t.Logf("wanted %+v", kat) + t.FailNow() + } + continue + } + } +} + +// TestKeccak does a basic test of the non-standardized Keccak hash functions. +func TestKeccak(t *testing.T) { + tests := []struct { + fn func() hash.Hash + data []byte + want string + }{ + { + NewLegacyKeccak256, + []byte("abc"), + "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", + }, + { + NewLegacyKeccak512, + []byte("abc"), + "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", + }, + } + + for _, u := range tests { + h := u.fn() + h.Write(u.data) + got := h.Sum(nil) + want := decodeHex(u.want) + if !bytes.Equal(got, want) { + t.Errorf("unexpected hash for size %d: got '%x' want '%s'", h.Size()*8, got, u.want) + } + } +} + +// TestUnalignedWrite tests that writing data in an arbitrary pattern with +// small input buffers. +func TestUnalignedWrite(t *testing.T) { + buf := sequentialBytes(0x10000) + for alg, df := range testDigests { + d := df() + d.Reset() + d.Write(buf) + want := d.Sum(nil) + d.Reset() + for i := 0; i < len(buf); { + // Cycle through offsets which make a 137 byte sequence. + // Because 137 is prime this sequence should exercise all corner cases. + offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1} + for _, j := range offsets { + if v := len(buf) - i; v < j { + j = v + } + d.Write(buf[i : i+j]) + i += j + } + } + got := d.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("Unaligned writes, alg=%s\ngot %q, want %q", alg, got, want) + } + } +} + +// sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing. +// +// The alignment of each slice is intentionally randomized to detect alignment +// issues in the implementation. See https://golang.org/issue/37644. +// Ideally, the compiler should fuzz the alignment itself. +// (See https://golang.org/issue/35128.) +func sequentialBytes(size int) []byte { + alignmentOffset := rand.Intn(8) + result := make([]byte, size+alignmentOffset)[alignmentOffset:] + for i := range result { + result[i] = byte(i) + } + return result +} + +func TestMarshalUnmarshal(t *testing.T) { + t.Run("Keccak-256", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak256()) }) + t.Run("Keccak-512", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak512()) }) +} + +// TODO(filippo): move this to crypto/internal/cryptotest. +func testMarshalUnmarshal(t *testing.T, h hash.Hash) { + buf := make([]byte, 200) + rand.Read(buf) + n := rand.Intn(200) + h.Write(buf) + want := h.Sum(nil) + h.Reset() + h.Write(buf[:n]) + b, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + t.Errorf("MarshalBinary: %v", err) + } + h.Write(bytes.Repeat([]byte{0}, 200)) + if err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary(b); err != nil { + t.Errorf("UnmarshalBinary: %v", err) + } + h.Write(buf[n:]) + got := h.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("got %x, want %x", got, want) + } +} + +// BenchmarkPermutationFunction measures the speed of the permutation function +// with no input data. +func BenchmarkPermutationFunction(b *testing.B) { + b.SetBytes(int64(200)) + var lanes [25]uint64 + for i := 0; i < b.N; i++ { + keccakF1600(&lanes) + } +} diff --git a/crypto/keccak/testdata/keccakKats.json.deflate b/crypto/keccak/testdata/keccakKats.json.deflate new file mode 100644 index 0000000000..7a94c2f8bc Binary files /dev/null and b/crypto/keccak/testdata/keccakKats.json.deflate differ diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 46509674b6..93d5f4ff94 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -143,9 +143,9 @@ func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index e9676ff1b8..03627ebafb 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -108,9 +108,9 @@ func gokzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index b82b147e3c..504602f5be 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -73,6 +73,10 @@ func (bitCurve *BitCurve) Params() *elliptic.CurveParams { // IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (bitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(bitCurve.P) >= 0 || y.Cmp(bitCurve.P) >= 0 { + return false + } + // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² y2.Mod(y2, bitCurve.P) //y²%P diff --git a/crypto/secp256k1/ext.h b/crypto/secp256k1/ext.h index 1c485e2603..baafb4404b 100644 --- a/crypto/secp256k1/ext.h +++ b/crypto/secp256k1/ext.h @@ -109,8 +109,10 @@ int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, unsigned char *point, ARG_CHECK(scalar != NULL); (void)ctx; - secp256k1_fe_set_b32_limit(&feX, point); - secp256k1_fe_set_b32_limit(&feY, point+32); + if (!secp256k1_fe_set_b32_limit(&feX, point) || + !secp256k1_fe_set_b32_limit(&feY, point+32)) { + return 0; + } secp256k1_ge_set_xy(&ge, &feX, &feY); secp256k1_scalar_set_b32(&s, scalar, &overflow); if (overflow || secp256k1_scalar_is_zero(&s)) { diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 014e166b7c..0dc360ee50 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -65,8 +65,8 @@ func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { // // The produced signature is in the [R || S || V] format where V is 0 or 1. func Sign(hash []byte, prv *ecdsa.PrivateKey) ([]byte, error) { - if len(hash) != 32 { - return nil, fmt.Errorf("hash is required to be exactly 32 bytes (%d)", len(hash)) + if len(hash) != DigestLength { + return nil, fmt.Errorf("hash is required to be exactly %d bytes (%d)", DigestLength, len(hash)) } if prv.Curve != S256() { return nil, errors.New("private key curve is not secp256k1") @@ -153,6 +153,13 @@ type btCurve struct { *secp256k1.KoblitzCurve } +func (curve btCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(secp256k1.Params().P) >= 0 || y.Cmp(secp256k1.Params().P) >= 0 { + return false + } + return curve.KoblitzCurve.IsOnCurve(x, y) +} + // Marshal converts a point given as (x, y) into a byte slice. func (curve btCurve) Marshal(x, y *big.Int) []byte { byteLen := (curve.Params().BitSize + 7) / 8 diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index 239a2134df..d11125c697 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -56,7 +56,7 @@ func Fuzz(data []byte) int { fmt.Printf("untrusted: %v\n", untrustedComment) fmt.Printf("trusted: %v\n", trustedComment) - err = SignifySignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) + err = SignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) if err != nil { panic(err) } @@ -68,7 +68,7 @@ func Fuzz(data []byte) int { signify = path } - _, err := exec.LookPath(signify) + _, err = exec.LookPath(signify) if err != nil { panic(err) } diff --git a/eth/api_backend.go b/eth/api_backend.go index e57d3c95d4..b58636dfc7 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -319,6 +319,11 @@ func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) e return b.eth.BlockChain().SubscribeChainHeadEvent(ch) } +// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent. +func (b *EthAPIBackend) SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription { + return b.eth.BlockChain().SubscribeNewPayloadEvent(ch) +} + func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return b.eth.BlockChain().SubscribeLogsEvent(ch) } diff --git a/eth/api_debug.go b/eth/api_debug.go index c5bf1c359e..4c8859072e 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -538,17 +538,14 @@ func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness if block == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) } - - result, err := bc.ProcessBlock(parent.Root, block, false, true) + result, err := bc.ProcessBlock(context.Background(), parent.Root, block, false, true) if err != nil { return &stateless.ExtWitness{}, err } - return result.Witness().ToExtWitness(), nil } @@ -558,16 +555,13 @@ func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWit if block == nil { return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) } - - result, err := bc.ProcessBlock(parent.Root, block, false, true) + result, err := bc.ProcessBlock(context.Background(), parent.Root, block, false, true) if err != nil { return &stateless.ExtWitness{}, err } - return result.Witness().ToExtWitness(), nil } diff --git a/eth/backend.go b/eth/backend.go index 6fded60a78..4f99732534 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -227,17 +227,19 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TriesInMemory: 128, TrieRetention: 30 * time.Minute, - TrieCleanLimit: config.TrieCleanCache, - NoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - ArchiveMode: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - MaxDiffLayers: config.MaxDiffLayers, - StateScheme: scheme, - ChainHistoryMode: config.HistoryMode, + TrieCleanLimit: config.TrieCleanCache, + NoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + ArchiveMode: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + MaxDiffLayers: config.MaxDiffLayers, + TrienodeHistory: config.TrienodeHistory, + NodeFullValueCheckpoint: config.NodeFullValueCheckpoint, + StateScheme: scheme, + ChainHistoryMode: config.HistoryMode, TxIndexer: &core.TxIndexerConfig{ Limit: min(config.TransactionHistory, math.MaxUint64), }, @@ -252,6 +254,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // - DATADIR/triedb/verkle.journal TrieJournalDirectory: stack.ResolvePath("triedb"), StateSizeTracking: config.EnableStateSizeTracking, + SlowBlockThreshold: config.SlowBlockThreshold, } ) if config.VMTrace != "" { @@ -500,6 +503,9 @@ func (s *Ethereum) updateFilterMapsHeads() { if head == nil || newHead.Hash() != head.Hash() { head = newHead chainView := s.newChainView(head) + if chainView == nil { + return + } historyCutoff, _ := s.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := s.blockchain.CurrentFinalBlock(); fb != nil { @@ -599,29 +605,3 @@ func (s *Ethereum) Stop() error { return nil } - -// SyncMode retrieves the current sync mode, either explicitly set, or derived -// from the chain status. -func (s *Ethereum) SyncMode() ethconfig.SyncMode { - // If we're in snap sync mode, return that directly - if s.handler.snapSync.Load() { - return ethconfig.SnapSync - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. - head := s.blockchain.CurrentBlock() - if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { - if head.Number.Uint64() < *pivot { - return ethconfig.SnapSync - } - } - // We are in a full sync, but the associated head state is missing. To complete - // the head state, forcefully rerun the snap sync. Note it doesn't mean the - // persistent state is corrupted, just mismatch with the head block. - if !s.blockchain.HasState(head.Root) { - log.Info("Reenabled snap sync as chain is stateless") - return ethconfig.SnapSync - } - // Nope, we're really full syncing - return ethconfig.FullSync -} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 6de9901bb3..b84d347d83 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -18,6 +18,7 @@ package catalyst import ( + "context" "errors" "fmt" "reflect" @@ -30,13 +31,14 @@ import ( "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -81,20 +83,6 @@ const ( beaconUpdateWarnFrequency = 5 * time.Minute ) -var ( - // Number of blobs requested via getBlobsV2 - getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) - - // Number of blobs requested via getBlobsV2 that are present in the blobpool - getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) - - // Number of times getBlobsV2 responded with “hit” - getBlobsV2RequestHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) - - // Number of times getBlobsV2 responded with “miss” - getBlobsV2RequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) -) - type ConsensusAPI struct { eth *eth.Ethereum @@ -137,6 +125,9 @@ type ConsensusAPI struct { // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. +// +// This function creates a long-lived object with an attached background thread. +// For testing or other short-term use cases, please use newConsensusAPIWithoutHeartbeat. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { api := newConsensusAPIWithoutHeartbeat(eth) go api.heartbeat() @@ -163,9 +154,6 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // // We try to set our blockchain to the headBlock. // -// If the method is called with an empty head block: we return success, which can be used -// to check if the engine API is enabled. -// // If the total difficulty was not reached: we return INVALID. // // If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, @@ -273,7 +261,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl } } log.Info("Forkchoice requested sync to new head", context...) - if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), header, finalized); err != nil { + if err := api.eth.Downloader().BeaconSync(header, finalized); err != nil { return engine.STATUS_SYNCING, err } return engine.STATUS_SYNCING, nil @@ -405,10 +393,12 @@ func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.Transit // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { - if !payloadID.Is(engine.PayloadV1) { - return nil, engine.UnsupportedFork - } - data, err := api.getPayload(payloadID, false) + data, err := api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1}, + nil, + ) if err != nil { return nil, err } @@ -417,35 +407,34 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.Execu // GetPayloadV2 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - // executionPayload: ExecutionPayloadV1 | ExecutionPayloadV2 where: - // - // - ExecutionPayloadV1 MUST be returned if the payload timestamp is lower - // than the Shanghai timestamp - // - // - ExecutionPayloadV2 MUST be returned if the payload timestamp is greater - // or equal to the Shanghai timestamp - if !payloadID.Is(engine.PayloadV1, engine.PayloadV2) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1, engine.PayloadV2}, + []forks.Fork{forks.Shanghai}, + ) } // GetPayloadV3 returns a cached payload by id. This endpoint should only // be used for the Cancun fork. func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Cancun}, + ) } // GetPayloadV4 returns a cached payload by id. This endpoint should only // be used for the Prague fork. func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Prague}, + ) } // GetPayloadV5 returns a cached payload by id. This endpoint should only @@ -454,18 +443,37 @@ func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.Execu // This method follows the same specification as engine_getPayloadV4 with // changes of returning BlobsBundleV2 with BlobSidecar version 1. func (api *ConsensusAPI) GetPayloadV5(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{ + forks.Osaka, + forks.BPO1, + forks.BPO2, + forks.BPO3, + forks.BPO4, + forks.BPO5, + }) } -func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool) (*engine.ExecutionPayloadEnvelope, error) { +// getPayload will retrieve the specified payload and verify it conforms to the +// endpoint's allowed payload versions and forks. +// +// Note passing nil `forks`, `versions` disables the respective check. +func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool, versions []engine.PayloadVersion, forks []forks.Fork) (*engine.ExecutionPayloadEnvelope, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) + if versions != nil && !payloadID.Is(versions...) { + return nil, engine.UnsupportedFork + } data := api.localBlocks.get(payloadID, full) if data == nil { return nil, engine.UnknownPayload } + if forks != nil && !api.checkFork(data.ExecutionPayload.Timestamp, forks...) { + return nil, engine.UnsupportedFork + } + return data, nil } @@ -500,7 +508,7 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { return nil, engine.InvalidParams.With(err) } @@ -546,8 +554,25 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { head := api.eth.BlockChain().CurrentHeader() if api.config().LatestFork(head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) < forks.Osaka { - return nil, unsupportedForkErr("engine_getBlobsV2 is not available before Osaka fork") + return nil, nil + } + return api.getBlobs(hashes, true) +} + +// GetBlobsV3 returns a set of blobs from the transaction pool. Same as +// GetBlobsV2, except will return partial responses in case there is a missing +// blob. +func (api *ConsensusAPI) GetBlobsV3(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { + head := api.eth.BlockChain().CurrentHeader() + if api.config().LatestFork(head.Time, types.DeserializeHeaderExtraInformation(head).ArbOSFormatVersion) < forks.Osaka { + return nil, nil } + return api.getBlobs(hashes, false) +} + +// getBlobs returns all available blobs. In v2, partial responses are not allowed, +// while v3 supports partial responses. +func (api *ConsensusAPI) getBlobs(hashes []common.Hash, v2 bool) ([]*engine.BlobAndProofV2, error) { if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } @@ -555,28 +580,30 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo getBlobsRequestedCounter.Inc(int64(len(hashes))) getBlobsAvailableCounter.Inc(int64(available)) - // Optimization: check first if all blobs are available, if not, return empty response - if available != len(hashes) { - getBlobsV2RequestMiss.Inc(1) + // Short circuit if partial response is not allowed + if v2 && available != len(hashes) { + getBlobsRequestMiss.Inc(1) return nil, nil } - - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1, false) + // Retrieve blobs from the pool. This operation is expensive and may involve + // heavy disk I/O. + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { return nil, engine.InvalidParams.With(err) } - - // To comply with API spec, check again that we really got all data needed - for _, blob := range blobs { - if blob == nil { - getBlobsV2RequestMiss.Inc(1) - return nil, nil - } - } - getBlobsV2RequestHit.Inc(1) - + // Validate the blobs from the pool and assemble the response res := make([]*engine.BlobAndProofV2, len(hashes)) - for i := 0; i < len(blobs); i++ { + for i := range blobs { + // The blob has been evicted since the last AvailableBlobs call. + // Return null if partial response is not allowed. + if blobs[i] == nil { + if !v2 { + continue + } else { + getBlobsRequestMiss.Inc(1) + return nil, nil + } + } var cellProofs []hexutil.Bytes for _, proof := range proofs[i] { cellProofs = append(cellProofs, proof[:]) @@ -586,6 +613,13 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo CellProofs: cellProofs, } } + if len(res) == len(hashes) { + getBlobsRequestCompleteHit.Inc(1) + } else if len(res) > 0 { + getBlobsRequestPartialHit.Inc(1) + } else { + getBlobsRequestMiss.Inc(1) + } return res, nil } @@ -593,15 +627,15 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo var invalidStatus = engine.PayloadStatusV1{Status: engine.INVALID} // NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return invalidStatus, paramsErr("withdrawals not supported in V1") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { arbosVersion := types.DeserializeHeaderExtraInformation(api.eth.BlockChain().CurrentHeader()).ArbOSFormatVersion var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp, arbosVersion) @@ -619,11 +653,11 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -638,11 +672,11 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, false) } // NewPayloadV4 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -663,10 +697,10 @@ func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHas if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, false) } -func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) newPayload(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (result engine.PayloadStatusV1, err error) { // The locking here is, strictly, not required. Without these locks, this can happen: // // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to @@ -680,6 +714,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // sequentially. // Hence, we use a lock here, to be sure that the previous call has finished before we // check whether we already have the block locally. + var attrs = []telemetry.Attribute{ + telemetry.Int64Attribute("block.number", int64(params.Number)), + telemetry.StringAttribute("block.hash", params.BlockHash.Hex()), + telemetry.Int64Attribute("tx.count", int64(len(params.Transactions))), + } + ctx, _, spanEnd := telemetry.StartSpan(ctx, "engine.newPayload", attrs...) + defer spanEnd(&err) api.newPayloadLock.Lock() defer api.newPayloadLock.Unlock() @@ -741,14 +782,14 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return api.delayPayloadImport(block), nil } if block.Time() <= parent.Time() { - log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) + log.Warn("Invalid timestamp", "parent", parent.Time(), "block", block.Time()) return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil } // Another corner case: if the node is in snap sync mode, but the CL client // tries to make it import a block. That should be denied as pushing something // into the database directly will conflict with the assumptions of snap sync // that it has an empty db that it can fill itself. - if api.eth.SyncMode() != ethconfig.FullSync { + if api.eth.Downloader().ConfigSyncMode() == ethconfig.SnapSync { return api.delayPayloadImport(block), nil } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { @@ -757,7 +798,9 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return engine.PayloadStatusV1{Status: engine.ACCEPTED}, nil } log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number()) - proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness) + start := time.Now() + proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(ctx, block, witness) + processingTime := time.Since(start) if err != nil { log.Warn("NewPayload: inserting block failed", "error", err) @@ -770,6 +813,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe } hash := block.Hash() + // Emit NewPayloadEvent for ethstats reporting + api.eth.BlockChain().SendNewPayloadEvent(core.NewPayloadEvent{ + Hash: hash, + Number: block.NumberU64(), + ProcessingTime: processingTime, + }) + // If witness collection was requested, inject that into the result too var ow *hexutil.Bytes if proofs != nil { @@ -796,16 +846,16 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadSt // Although we don't want to trigger a sync, if there is one already in // progress, try to extend it with the current payload request to relieve // some strain from the forkchoice update. - err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()) + err := api.eth.Downloader().BeaconExtend(block.Header()) if err == nil { log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) return engine.PayloadStatusV1{Status: engine.SYNCING} } // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just + // payload as non-integrate on top of the existing sync. We'll just // have to rely on the beacon client to forcefully update the head with // a forkchoice update request. - if api.eth.SyncMode() == ethconfig.FullSync { + if api.eth.Downloader().ConfigSyncMode() == ethconfig.FullSync { // In full sync mode, failure to import a well-formed block can only mean // that the parent state is missing and the syncer rejected extending the // current cycle with the new payload. @@ -900,8 +950,6 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) engine.Pa // heartbeat loops indefinitely, and checks if there have been beacon client updates // received in the last while. If not - or if they but strange ones - it warns the // user that something might be off with their consensus node. -// -// TODO(karalabe): Spin this goroutine down somehow func (api *ConsensusAPI) heartbeat() { // Sleep a bit on startup since there's obviously no beacon client yet // attached, so no need to print scary warnings to the user. diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 793948eebb..abe13bbd2c 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -203,7 +203,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV1, }).Id() - execData, err := api.getPayload(payloadID, true) + execData, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -314,7 +314,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) switch { case err != nil: t.Fatalf("Failed to insert block: %v", err) @@ -356,7 +356,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } @@ -426,7 +426,7 @@ func TestEth2DeepReorg(t *testing.T) { } // startEthService creates a full node instance for testing. -func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { +func startEthService(t testing.TB, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { t.Helper() n, err := node.New(&node.Config{ @@ -502,7 +502,7 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.He } envelope := getNewEnvelope(t, api, parent, w, h) - execResp, err := api.newPayload(*envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) + execResp, err := api.newPayload(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -636,7 +636,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { t.Fatalf("error preparing payload, invalid status: %v", resp.PayloadStatus.Status) } // give the payload some time to be built - if payload, err = api.getPayload(*resp.PayloadID, true); err != nil { + if payload, err = api.getPayload(*resp.PayloadID, true, nil, nil); err != nil { t.Fatalf("can't get payload: %v", err) } if len(payload.ExecutionPayload.Transactions) > 0 { @@ -648,7 +648,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { t.Fatalf("payload should not be empty") } } - execResp, err := api.NewPayloadV1(*payload.ExecutionPayload) + execResp, err := api.NewPayloadV1(context.Background(), *payload.ExecutionPayload) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -708,7 +708,7 @@ func TestEmptyBlocks(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -724,7 +724,7 @@ func TestEmptyBlocks(t *testing.T) { payload.GasUsed += 1 payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -742,7 +742,7 @@ func TestEmptyBlocks(t *testing.T) { payload.ParentHash = common.Hash{1} payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -859,7 +859,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { // feed the payloads to node B for _, payload := range invalidChain { - status, err := apiB.NewPayloadV1(*payload) + status, err := apiB.NewPayloadV1(context.Background(), *payload) if err != nil { panic(err) } @@ -892,7 +892,7 @@ func TestInvalidBloom(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) payload.LogsBloom = append(payload.LogsBloom, byte(1)) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -931,7 +931,7 @@ func TestSimultaneousNewBlock(t *testing.T) { for ii := 0; ii < 10; ii++ { go func() { defer wg.Done() - if newResp, err := api.NewPayloadV1(*execData); err != nil { + if newResp, err := api.NewPayloadV1(context.Background(), *execData); err != nil { errMu.Lock() testErr = fmt.Errorf("failed to insert block: %w", err) errMu.Unlock() @@ -1038,7 +1038,7 @@ func TestWithdrawals(t *testing.T) { } // 10: verify locally built block - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1082,7 +1082,7 @@ func TestWithdrawals(t *testing.T) { if err != nil { t.Fatalf("error getting payload, err=%v", err) } - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1219,15 +1219,15 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, Version: payloadVersion, }).Id() - execData, err := api.GetPayloadV2(payloadID) + execData, err := api.getPayload(payloadID, false, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } var status engine.PayloadStatusV1 if !shanghai { - status, err = api.NewPayloadV1(*execData.ExecutionPayload) + status, err = api.NewPayloadV1(context.Background(), *execData.ExecutionPayload) } else { - status, err = api.NewPayloadV2(*execData.ExecutionPayload) + status, err = api.NewPayloadV2(context.Background(), *execData.ExecutionPayload) } if err != nil { t.Fatalf("error validating payload: %v", err.(*engine.EngineAPIError).ErrorData()) @@ -1598,7 +1598,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { } // 11: verify locally built block - if status, err := api.NewPayloadV3(*execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { + if status, err := api.NewPayloadV3(context.Background(), *execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1674,7 +1674,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV3, }).Id() - envelope, err := api.getPayload(payloadID, true) + envelope, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -1705,7 +1705,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { envelope.ExecutionPayload.StateRoot = wantStateRoot envelope.ExecutionPayload.ReceiptsRoot = wantReceiptRoot - res2, err := api.NewPayloadWithWitnessV3(*envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) + res2, err := api.NewPayloadWithWitnessV3(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) if err != nil { t.Fatalf("error executing stateless payload witness: %v", err) } @@ -1873,7 +1873,7 @@ func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount in return types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx) } -func newGetBlobEnv(t *testing.T, version byte) (*node.Node, *ConsensusAPI) { +func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) { var ( // Create a database pre-initialize with a genesis block config = *params.MergedTestChainConfig @@ -2016,7 +2016,7 @@ func TestGetBlobsV1AfterOsakaFork(t *testing.T) { } } -func TestGetBlobsV2(t *testing.T) { +func TestGetBlobsV2And3(t *testing.T) { n, api := newGetBlobEnv(t, 1) defer n.Close() @@ -2045,36 +2045,64 @@ func TestGetBlobsV2(t *testing.T) { }, } for i, suite := range suites { - // Fill the request for retrieving blobs - var ( - vhashes []common.Hash - expect []*engine.BlobAndProofV2 - ) - // fill missing blob - if suite.fillRandom { - vhashes = append(vhashes, testrand.Hash()) - } - for j := suite.start; j < suite.limit; j++ { - vhashes = append(vhashes, testBlobVHashes[j]) - var cellProofs []hexutil.Bytes - for _, proof := range testBlobCellProofs[j] { - cellProofs = append(cellProofs, proof[:]) + runGetBlobs(t, api.GetBlobsV2, suite.start, suite.limit, suite.fillRandom, false, fmt.Sprintf("GetBlobsV2 suite=%d", i)) + runGetBlobs(t, api.GetBlobsV3, suite.start, suite.limit, suite.fillRandom, true, fmt.Sprintf("GetBlobsV3 suite=%d %v", i, suite)) + } +} + +// Benchmark GetBlobsV2 internals +// Note that this is not an RPC-level benchmark, so JSON-RPC overhead is not included. +func BenchmarkGetBlobsV2(b *testing.B) { + n, api := newGetBlobEnv(b, 1) + defer n.Close() + + // for blobs in [1, 2, 4, 6], print string and run benchmark + for _, blobs := range []int{1, 2, 4, 6} { + name := fmt.Sprintf("blobs=%d", blobs) + b.Run(name, func(b *testing.B) { + for b.Loop() { + runGetBlobs(b, api.GetBlobsV2, 0, blobs, false, false, name) } - expect = append(expect, &engine.BlobAndProofV2{ - Blob: testBlobs[j][:], - CellProofs: cellProofs, - }) - } - result, err := api.GetBlobsV2(vhashes) - if err != nil { - t.Errorf("Unexpected error for case %d, %v", i, err) - } - // null is responded if any blob is missing - if suite.fillRandom { + }) + } +} + +type getBlobsFn func(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) + +func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom bool, expectPartialResponse bool, name string) { + // Fill the request for retrieving blobs + var ( + vhashes []common.Hash + expect []*engine.BlobAndProofV2 + ) + for j := start; j < limit; j++ { + vhashes = append(vhashes, testBlobVHashes[j]) + var cellProofs []hexutil.Bytes + for _, proof := range testBlobCellProofs[j] { + cellProofs = append(cellProofs, proof[:]) + } + expect = append(expect, &engine.BlobAndProofV2{ + Blob: testBlobs[j][:], + CellProofs: cellProofs, + }) + } + // fill missing blob + if fillRandom { + vhashes = append(vhashes, testrand.Hash()) + } + result, err := getBlobs(vhashes) + if err != nil { + t.Errorf("Unexpected error for case %s, %v", name, err) + } + if fillRandom { + if expectPartialResponse { + expect = append(expect, nil) + } else { + // Nil is expected if getBlobs can not return a partial response expect = nil } - if !reflect.DeepEqual(result, expect) { - t.Fatalf("Unexpected result for case %d", i) - } + } + if !reflect.DeepEqual(result, expect) { + t.Fatalf("Unexpected result for case %s", name) } } diff --git a/eth/catalyst/metrics.go b/eth/catalyst/metrics.go new file mode 100644 index 0000000000..01a24191b0 --- /dev/null +++ b/eth/catalyst/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + // Number of blobs requested via getBlobsV2 + getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) + + // Number of blobs requested via getBlobsV2 that are present in the blobpool + getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) + + // Number of times getBlobsV2/V3 responded with all blobs + getBlobsRequestCompleteHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) + + // Number of times getBlobsV2/V3 responded with no blobs. V2 will return no + // blobs if it doesn't have all the blobs (all or nothing). + getBlobsRequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) + + // Number of times getBlobsV3 responded with some, but not all, blobs + getBlobsRequestPartialHit = metrics.NewRegisteredCounter("engine/getblobs/partial", nil) +) diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 6fee56d37b..5d7bb87170 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "crypto/rand" "crypto/sha256" "errors" @@ -32,11 +33,13 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" + "go.opentelemetry.io/otel" ) const devEpochLength = 32 @@ -196,6 +199,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } version := payloadVersion(c.eth.BlockChain().Config(), timestamp, c.eth.BlockChain().CurrentBlock()) + tracer := otel.Tracer("") var random [32]byte rand.Read(random[:]) @@ -219,7 +223,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u return nil } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) + envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true, nil, nil) if err != nil { return err } @@ -260,8 +264,16 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u requests = envelope.Requests } + // Create a server span for newPayload, simulating the consensus client + // sending the execution payload for validation. + npCtx, npSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "newPayloadV" + fmt.Sprintf("%d", version), + }) // Mark the payload as canon - _, err = c.engineAPI.newPayload(*payload, blobHashes, beaconRoot, requests, false) + _, err = c.engineAPI.newPayload(npCtx, *payload, blobHashes, beaconRoot, requests, false) + npSpanEnd(&err) if err != nil { return err } @@ -289,9 +301,8 @@ func (c *SimulatedBeacon) loop() { case <-timer.C: if err := c.sealBlock(c.withdrawals.pop(10), uint64(time.Now().Unix())); err != nil { log.Warn("Error performing sealing work", "err", err) - } else { - timer.Reset(time.Second * time.Duration(c.period)) } + timer.Reset(time.Second * time.Duration(c.period)) } } } diff --git a/eth/catalyst/witness.go b/eth/catalyst/witness.go index a519516036..57827f3a2a 100644 --- a/eth/catalyst/witness.go +++ b/eth/catalyst/witness.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "errors" "strconv" "time" @@ -87,16 +88,16 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.Forkchoice // NewPayloadWithWitnessV1 is analogous to NewPayloadV1, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV2 is analogous to NewPayloadV2, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { arbosVersion := types.DeserializeHeaderExtraInformation(api.eth.BlockChain().CurrentHeader()).ArbOSFormatVersion var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp, arbosVersion) @@ -114,12 +115,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) ( case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV3 is analogous to NewPayloadV3, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -134,12 +135,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, v case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, true) } // NewPayloadWithWitnessV4 is analogous to NewPayloadV4, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -160,7 +161,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, v if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, true) } // ExecuteStatelessPayloadV1 is analogous to NewPayloadV1, only it operates in @@ -286,7 +287,7 @@ func (api *ConsensusAPI) executeStatelessPayload(params engine.ExecutableData, v api.lastNewPayloadUpdate.Store(time.Now().Unix()) log.Trace("Executing block statelessly", "number", block.Number(), "hash", params.BlockHash) - stateRoot, receiptRoot, err := core.ExecuteStateless(api.config(), vm.Config{}, block, witness) + stateRoot, receiptRoot, err := core.ExecuteStateless(context.Background(), api.config(), vm.Config{}, block, witness) if err != nil { log.Warn("ExecuteStatelessPayload: execution failed", "err", err) errorMsg := err.Error() diff --git a/eth/downloader/beacondevsync.go b/eth/downloader/beacondevsync.go index 03f17b1a52..52e43f86b4 100644 --- a/eth/downloader/beacondevsync.go +++ b/eth/downloader/beacondevsync.go @@ -33,14 +33,14 @@ import ( // Note, this must not be used in live code. If the forkchcoice endpoint where // to use this instead of giving us the payload first, then essentially nobody // in the network would have the block yet that we'd attempt to retrieve. -func (d *Downloader) BeaconDevSync(mode SyncMode, header *types.Header) error { +func (d *Downloader) BeaconDevSync(header *types.Header) error { // Be very loud that this code should not be used in a live node log.Warn("----------------------------------") log.Warn("Beacon syncing with hash as target", "number", header.Number, "hash", header.Hash()) log.Warn("This is unhealthy for a live node!") log.Warn("This is incompatible with the consensus layer!") log.Warn("----------------------------------") - return d.BeaconSync(mode, header, header) + return d.BeaconSync(header, header) } // GetHeader tries to retrieve the header with a given hash from a random peer. diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 12b74a1ba9..914e1dfada 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -34,10 +34,8 @@ import ( // directed by the skeleton sync's head/tail events. type beaconBackfiller struct { downloader *Downloader // Downloader to direct via this callback implementation - syncMode SyncMode // Sync mode to use for backfilling the skeleton chains success func() // Callback to run on successful sync cycle completion filling bool // Flag whether the downloader is backfilling or not - filled *types.Header // Last header filled by the last terminated sync loop started chan struct{} // Notification channel whether the downloader inited lock sync.Mutex // Mutex protecting the sync lock } @@ -57,12 +55,15 @@ func (b *beaconBackfiller) suspend() *types.Header { // If no filling is running, don't waste cycles b.lock.Lock() filling := b.filling - filled := b.filled started := b.started b.lock.Unlock() if !filling { - return filled // Return the filled header on the previous sync completion + // Sync cycle was inactive, retrieve and return the latest snap block + // as the filled header. + log.Debug("Backfiller was inactive") + + return b.downloader.blockchain.CurrentSnapBlock() } // A previous filling should be running, though it may happen that it hasn't // yet started (being done on a new goroutine). Many concurrent beacon head @@ -74,9 +75,9 @@ func (b *beaconBackfiller) suspend() *types.Header { // Now that we're sure the downloader successfully started up, we can cancel // it safely without running the risk of data races. b.downloader.Cancel() + log.Debug("Backfiller has been suspended") // Sync cycle was just terminated, retrieve and return the last filled header. - // Can't use `filled` as that contains a stale value from before cancellation. return b.downloader.blockchain.CurrentSnapBlock() } @@ -87,12 +88,11 @@ func (b *beaconBackfiller) resume() { // If a previous filling cycle is still running, just ignore this start // request. // TODO(karalabe): We should make this channel driven b.lock.Unlock() + log.Debug("Backfiller is running") return } b.filling = true - b.filled = nil b.started = make(chan struct{}) - mode := b.syncMode b.lock.Unlock() // Start the backfilling on its own thread since the downloader does not have @@ -102,42 +102,22 @@ func (b *beaconBackfiller) resume() { defer func() { b.lock.Lock() b.filling = false - b.filled = b.downloader.blockchain.CurrentSnapBlock() b.lock.Unlock() }() // If the downloader fails, report an error as in beacon chain mode there // should be no errors as long as the chain we're syncing to is valid. - if err := b.downloader.synchronise(mode, b.started); err != nil { + if err := b.downloader.synchronise(b.started); err != nil { log.Error("Beacon backfilling failed", "err", err) return } // Synchronization succeeded. Since this happens async, notify the outer - // context to disable snap syncing and enable transaction propagation. + // context to enable transaction propagation. if b.success != nil { b.success() } + log.Debug("Backfilling completed") }() -} - -// setMode updates the sync mode from the current one to the requested one. If -// there's an active sync in progress, it will be cancelled and restarted. -func (b *beaconBackfiller) setMode(mode SyncMode) { - // Update the old sync mode and track if it was changed - b.lock.Lock() - oldMode := b.syncMode - updated := oldMode != mode - filling := b.filling - b.syncMode = mode - b.lock.Unlock() - - // If the sync mode was changed mid-sync, restart. This should never ever - // really happen, we just handle it to detect programming errors. - if !updated || !filling { - return - } - log.Error("Downloader sync mode changed mid-run", "old", oldMode.String(), "new", mode.String()) - b.suspend() - b.resume() + log.Debug("Backfilling started") } // SetBadBlockCallback sets the callback to run when a bad block is hit by the @@ -153,8 +133,8 @@ func (d *Downloader) SetBadBlockCallback(onBadBlock badBlockFn) { // // Internally backfilling and state sync is done the same way, but the header // retrieval and scheduling is replaced. -func (d *Downloader) BeaconSync(mode SyncMode, head *types.Header, final *types.Header) error { - return d.beaconSync(mode, head, final, true) +func (d *Downloader) BeaconSync(head *types.Header, final *types.Header) error { + return d.beaconSync(head, final, true) } // BeaconExtend is an optimistic version of BeaconSync, where an attempt is made @@ -163,8 +143,8 @@ func (d *Downloader) BeaconSync(mode SyncMode, head *types.Header, final *types. // // This is useful if a beacon client is feeding us large chunks of payloads to run, // but is not setting the head after each. -func (d *Downloader) BeaconExtend(mode SyncMode, head *types.Header) error { - return d.beaconSync(mode, head, nil, false) +func (d *Downloader) BeaconExtend(head *types.Header) error { + return d.beaconSync(head, nil, false) } // beaconSync is the post-merge version of the chain synchronization, where the @@ -173,20 +153,9 @@ func (d *Downloader) BeaconExtend(mode SyncMode, head *types.Header) error { // // Internally backfilling and state sync is done the same way, but the header // retrieval and scheduling is replaced. -func (d *Downloader) beaconSync(mode SyncMode, head *types.Header, final *types.Header, force bool) error { - // When the downloader starts a sync cycle, it needs to be aware of the sync - // mode to use (full, snap). To keep the skeleton chain oblivious, inject the - // mode into the backfiller directly. - // - // Super crazy dangerous type cast. Should be fine (TM), we're only using a - // different backfiller implementation for skeleton tests. - d.skeleton.filler.(*beaconBackfiller).setMode(mode) - +func (d *Downloader) beaconSync(head *types.Header, final *types.Header, force bool) error { // Signal the skeleton sync to switch to a new head, however it wants - if err := d.skeleton.Sync(head, final, force); err != nil { - return err - } - return nil + return d.skeleton.Sync(head, final, force) } // findBeaconAncestor tries to locate the common ancestor link of the local chain @@ -217,6 +186,8 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { log.Error("Failed to retrieve beacon bounds", "err", err) return 0, err } + log.Debug("Searching beacon ancestor", "local", number, "beaconhead", beaconHead.Number, "beacontail", beaconTail.Number) + var linked bool switch d.getMode() { case ethconfig.FullSync: @@ -270,6 +241,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { } start = check } + log.Debug("Found beacon ancestor", "number", start) return start, nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 09837a3045..caeb3d64dd 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -97,8 +97,9 @@ type headerTask struct { } type Downloader struct { - mode atomic.Uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode - mux *event.TypeMux // Event multiplexer to announce sync operation events + mode atomic.Uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode + moder *syncModer // Sync mode management, deliver the appropriate sync mode choice for each cycle + mux *event.TypeMux // Event multiplexer to announce sync operation events queue *queue // Scheduler for selecting the hashes to download peers *peerSet // Set of active peers from which download can proceed @@ -165,6 +166,9 @@ type BlockChain interface { // HasHeader verifies a header's presence in the local chain. HasHeader(common.Hash, uint64) bool + // HasState checks if state trie is fully present in the database or not. + HasState(root common.Hash) bool + // GetHeaderByHash retrieves a header from the local chain. GetHeaderByHash(common.Hash) *types.Header @@ -189,8 +193,12 @@ type BlockChain interface { // CurrentSnapBlock retrieves the head snap block from the local chain. CurrentSnapBlock() *types.Header - // SnapSyncCommitHead directly commits the head block to a certain entity. - SnapSyncCommitHead(common.Hash) error + // SnapSyncStart explicitly notifies the chain that snap sync is scheduled and + // marks chain mutations as disallowed. + SnapSyncStart() error + + // SnapSyncComplete directly commits the head block to a certain entity. + SnapSyncComplete(common.Hash) error // InsertHeadersBeforeCutoff inserts a batch of headers before the configured // chain cutoff into the ancient store. @@ -221,10 +229,11 @@ type BlockChain interface { } // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader { +func New(stateDb ethdb.Database, mode ethconfig.SyncMode, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader { cutoffNumber, cutoffHash := chain.HistoryPruningCutoff() dl := &Downloader{ stateDB: stateDb, + moder: newSyncModer(mode, chain, stateDb), mux: mux, queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), peers: newPeerSet(), @@ -239,7 +248,7 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(), } // Create the post-merge skeleton syncer and start the process - dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success)) + dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success), chain) go dl.stateFetcher() return dl @@ -331,7 +340,7 @@ func (d *Downloader) UnregisterPeer(id string) error { // synchronise will select the peer and use it for synchronising. If an empty string is given // it will use the best peer possible and synchronize if its TD is higher than our own. If any of the // checks fail an error will be returned. This method is synchronous -func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error { +func (d *Downloader) synchronise(beaconPing chan struct{}) (err error) { // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and @@ -356,21 +365,21 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error if d.notified.CompareAndSwap(false, true) { log.Info("Block synchronisation started") } - if mode == ethconfig.SnapSync { - // Snap sync will directly modify the persistent state, making the entire - // trie database unusable until the state is fully synced. To prevent any - // subsequent state reads, explicitly disable the trie database and state - // syncer is responsible to address and correct any state missing. - if d.blockchain.TrieDB().Scheme() == rawdb.PathScheme { - if err := d.blockchain.TrieDB().Disable(); err != nil { - return err - } + + // Obtain the synchronized used in this cycle + mode := d.moder.get(true) + defer func() { + if err == nil && mode == ethconfig.SnapSync { + d.moder.disableSnap() + log.Info("Disabled snap-sync after the initial sync cycle") } - // Snap sync uses the snapshot namespace to store potentially flaky data until - // sync completely heals and finishes. Pause snapshot maintenance in the mean- - // time to prevent access. - if snapshots := d.blockchain.Snapshots(); snapshots != nil { // Only nil in tests - snapshots.Disable() + }() + + // Disable chain mutations when snap sync is selected, ensuring the + // downloader is the sole mutator. + if mode == ethconfig.SnapSync { + if err := d.blockchain.SnapSyncStart(); err != nil { + return err } } // Reset the queue, peer set and wake channels to clean any internal leftover state @@ -399,6 +408,7 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error // Atomically set the requested sync mode d.mode.Store(uint32(mode)) + defer d.mode.Store(0) if beaconPing != nil { close(beaconPing) @@ -406,10 +416,17 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error return d.syncToHead() } +// getMode returns the sync mode used within current cycle. func (d *Downloader) getMode() SyncMode { return SyncMode(d.mode.Load()) } +// ConfigSyncMode returns the sync mode configured for the node. +// The actual running sync mode can differ from this. +func (d *Downloader) ConfigSyncMode() SyncMode { + return d.moder.get(false) +} + // syncToHead starts a block synchronization based on the hash chain from // the specified head hash. func (d *Downloader) syncToHead() (err error) { @@ -1066,7 +1083,7 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error { if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []rlp.RawValue{result.Receipts}, d.ancientLimit); err != nil { return err } - if err := d.blockchain.SnapSyncCommitHead(block.Hash()); err != nil { + if err := d.blockchain.SnapSyncComplete(block.Hash()); err != nil { return err } d.committed.Store(true) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 12d33495d2..91684309e6 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/event" @@ -49,12 +50,12 @@ type downloadTester struct { } // newTester creates a new downloader test mocker. -func newTester(t *testing.T) *downloadTester { - return newTesterWithNotification(t, nil) +func newTester(t *testing.T, mode ethconfig.SyncMode) *downloadTester { + return newTesterWithNotification(t, mode, nil) } // newTesterWithNotification creates a new downloader test mocker. -func newTesterWithNotification(t *testing.T, success func()) *downloadTester { +func newTesterWithNotification(t *testing.T, mode ethconfig.SyncMode, success func()) *downloadTester { db, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) if err != nil { panic(err) @@ -75,7 +76,7 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { chain: chain, peers: make(map[string]*downloadTesterPeer), } - tester.downloader = New(db, new(event.TypeMux), tester.chain, tester.dropPeer, success) + tester.downloader = New(db, mode, new(event.TypeMux), tester.chain, tester.dropPeer, success) return tester } @@ -213,10 +214,12 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { blobs := eth.ServiceGetBlockBodiesQuery(dlp.chain, hashes) - bodies := make([]*eth.BlockBody, len(blobs)) + bodies := make([]*types.Body, len(blobs)) + ethbodies := make([]eth.BlockBody, len(blobs)) for i, blob := range blobs { - bodies[i] = new(eth.BlockBody) + bodies[i] = new(types.Body) rlp.DecodeBytes(blob, bodies[i]) + rlp.DecodeBytes(blob, ðbodies[i]) } var ( txsHashes = make([]common.Hash, len(bodies)) @@ -238,9 +241,13 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *et Peer: dlp.id, } res := ð.Response{ - Req: req, - Res: (*eth.BlockBodiesResponse)(&bodies), - Meta: [][]common.Hash{txsHashes, uncleHashes, withdrawalHashes}, + Req: req, + Res: (*eth.BlockBodiesResponse)(ðbodies), + Meta: eth.BlockBodyHashes{ + TransactionRoots: txsHashes, + UncleHashes: uncleHashes, + WithdrawalRoots: withdrawalHashes, + }, Time: 1, Done: make(chan error, 1), // Ignore the returned status } @@ -289,14 +296,14 @@ func (dlp *downloadTesterPeer) ID() string { // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. -func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error { // Create the request and service it req := &snap.GetAccountRangePacket{ ID: id, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), } slimaccs, proofs := snap.ServiceGetAccountRangeQuery(dlp.chain, req) @@ -315,7 +322,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi // RequestStorageRanges fetches a batch of storage slots belonging to one or // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. -func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { // Create the request and service it req := &snap.GetStorageRangesPacket{ ID: id, @@ -323,7 +330,7 @@ func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), } storage, proofs := snap.ServiceGetStorageRangesQuery(dlp.chain, req) @@ -340,25 +347,28 @@ func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, } // RequestByteCodes fetches a batch of bytecodes by hash. -func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { req := &snap.GetByteCodesPacket{ ID: id, Hashes: hashes, - Bytes: bytes, + Bytes: uint64(bytes), } codes := snap.ServiceGetByteCodesQuery(dlp.chain, req) go dlp.dl.downloader.SnapSyncer.OnByteCodes(dlp, id, codes) return nil } -// RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specific state trie. -func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { +// RequestTrieNodes fetches a batch of account or storage trie nodes. +func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []snap.TrieNodePathSet, bytes int) error { + encPaths, err := rlp.EncodeToRawList(paths) + if err != nil { + panic(err) + } req := &snap.GetTrieNodesPacket{ ID: id, Root: root, - Paths: paths, - Bytes: bytes, + Paths: encPaths, + Bytes: uint64(bytes), } nodes, _ := snap.ServiceGetTrieNodesQuery(dlp.chain, req, time.Now()) go dlp.dl.downloader.SnapSyncer.OnTrieNodes(dlp, id, nodes) @@ -394,7 +404,7 @@ func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -404,7 +414,7 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester.newPeer("peer", protocol, chain.blocks[1:]) // Synchronise with the peer and make sure all relevant data was retrieved - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } select { @@ -422,7 +432,7 @@ func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) // func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) + tester := newTester(t, mode) defer tester.terminate() // Create a long block chain to download and the tester @@ -439,7 +449,7 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Start a synchronisation concurrently errc := make(chan error, 1) go func() { - errc <- tester.downloader.BeaconSync(mode, testChainBase.blocks[len(testChainBase.blocks)-1].Header(), nil) + errc <- tester.downloader.BeaconSync(testChainBase.blocks[len(testChainBase.blocks)-1].Header(), nil) }() // Iteratively take some blocks, always checking the retrieval count for { @@ -505,7 +515,7 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { success := func() { close(complete) } - tester := newTesterWithNotification(t, success) + tester := newTesterWithNotification(t, mode, success) defer tester.terminate() chain := testChainBase.shorten(MaxHeaderFetch) @@ -517,7 +527,7 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { t.Errorf("download queue not idle") } // Synchronise with the peer, but cancel afterwards - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } <-complete @@ -538,7 +548,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { success := func() { close(complete) } - tester := newTesterWithNotification(t, success) + tester := newTesterWithNotification(t, mode, success) defer tester.terminate() // Create a small enough block chain to download @@ -547,7 +557,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to start beacon sync: %v", err) } select { @@ -575,7 +585,7 @@ func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ET func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -593,7 +603,7 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { receiptsHave.Add(int32(len(headers))) } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } select { @@ -656,7 +666,7 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { close(success) }) defer tester.terminate() @@ -668,7 +678,7 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { if c.local > 0 { tester.chain.InsertChain(chain.blocks[1 : c.local+1]) } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("Failed to beacon sync chain %v %v", c.name, err) } select { @@ -692,7 +702,7 @@ func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullS func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) - tester := newTesterWithNotification(t, func() { + tester := newTesterWithNotification(t, mode, func() { success <- struct{}{} }) defer tester.terminate() @@ -707,7 +717,7 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { faultyPeer.withholdBodies[header.Hash()] = struct{}{} } - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)/2-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)/2-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } select { @@ -723,7 +733,7 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Synchronise all the blocks and check continuation progress tester.newPeer("peer-full", protocol, chain.blocks[1:]) - if err := tester.downloader.BeaconSync(mode, chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { + if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { t.Fatalf("failed to beacon-sync chain: %v", err) } startingBlock := uint64(len(chain.blocks)/2 - 1) diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index 56359b33c9..6a8eb35219 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -88,15 +88,14 @@ func (q *bodyQueue) request(peer *peerConnection, req *fetchRequest, resCh chan // deliver is responsible for taking a generic response packet from the concurrent // fetcher, unpacking the body data and delivering it to the downloader's queue. func (q *bodyQueue) deliver(peer *peerConnection, packet *eth.Response) (int, error) { - txs, uncles, withdrawals := packet.Res.(*eth.BlockBodiesResponse).Unpack() - hashsets := packet.Meta.([][]common.Hash) // {txs hashes, uncle hashes, withdrawal hashes} - - accepted, err := q.queue.DeliverBodies(peer.id, txs, hashsets[0], uncles, hashsets[1], withdrawals, hashsets[2]) + resp := packet.Res.(*eth.BlockBodiesResponse) + meta := packet.Meta.(eth.BlockBodyHashes) + accepted, err := q.queue.DeliverBodies(peer.id, meta, *resp) switch { - case err == nil && len(txs) == 0: + case err == nil && len(*resp) == 0: peer.log.Trace("Requested bodies delivered") case err == nil: - peer.log.Trace("Delivered new batch of bodies", "count", len(txs), "accepted", accepted) + peer.log.Trace("Delivered new batch of bodies", "count", len(*resp), "accepted", accepted) default: peer.log.Debug("Failed to deliver retrieved bodies", "err", err) } diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 9fe169d5f7..c0cb9b174a 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -29,11 +29,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -418,7 +417,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common skip := make([]*types.Header, 0) progress := false throttled := false - for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { + for len(send) < count && !taskQueue.Empty() { // the task queue will pop items in order, so the highest prio block // is also the lowest block number. header, _ := taskQueue.Peek() @@ -433,7 +432,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common taskQueue.PopItem() progress = true delete(taskPool, header.Hash()) - proc = proc - 1 log.Error("Fetch reservation already delivered", "number", header.Number.Uint64()) continue } @@ -455,7 +453,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common // If it's a noop, we can skip this task delete(taskPool, header.Hash()) taskQueue.PopItem() - proc = proc - 1 progress = true continue } @@ -561,63 +558,54 @@ func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue // DeliverBodies injects a block body retrieval response into the results queue. // The method returns the number of blocks bodies accepted from the delivery and // also wakes any threads waiting for data delivery. -func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListHashes []common.Hash, - uncleLists [][]*types.Header, uncleListHashes []common.Hash, - withdrawalLists [][]*types.Withdrawal, withdrawalListHashes []common.Hash, -) (int, error) { +func (q *queue) DeliverBodies(id string, hashes eth.BlockBodyHashes, bodies []eth.BlockBody) (int, error) { q.lock.Lock() defer q.lock.Unlock() + var txLists [][]*types.Transaction + var uncleLists [][]*types.Header + var withdrawalLists [][]*types.Withdrawal + validate := func(index int, header *types.Header) error { - if txListHashes[index] != header.TxHash { + if hashes.TransactionRoots[index] != header.TxHash { return errInvalidBody } - if uncleListHashes[index] != header.UncleHash { + if hashes.UncleHashes[index] != header.UncleHash { return errInvalidBody } if header.WithdrawalsHash == nil { // nil hash means that withdrawals should not be present in body - if withdrawalLists[index] != nil { + if bodies[index].Withdrawals != nil { return errInvalidBody } } else { // non-nil hash: body must have withdrawals - if withdrawalLists[index] == nil { + if bodies[index].Withdrawals == nil { return errInvalidBody } - if withdrawalListHashes[index] != *header.WithdrawalsHash { + if hashes.WithdrawalRoots[index] != *header.WithdrawalsHash { return errInvalidBody } } - // Blocks must have a number of blobs corresponding to the header gas usage, - // and zero before the Cancun hardfork. - var blobs int - for _, tx := range txLists[index] { - // Count the number of blobs to validate against the header's blobGasUsed - blobs += len(tx.BlobHashes()) - - // Validate the data blobs individually too - if tx.Type() == types.BlobTxType { - if len(tx.BlobHashes()) == 0 { - return errInvalidBody - } - for _, hash := range tx.BlobHashes() { - if !kzg4844.IsValidVersionedHash(hash[:]) { - return errInvalidBody - } - } - if tx.BlobTxSidecar() != nil { - return errInvalidBody - } - } + + // decode + txs, err := bodies[index].Transactions.Items() + if err != nil { + return fmt.Errorf("%w: bad transactions: %v", errInvalidBody, err) } - if header.BlobGasUsed != nil { - if want := *header.BlobGasUsed / params.BlobTxBlobGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated - return errInvalidBody + txLists = append(txLists, txs) + uncles, err := bodies[index].Uncles.Items() + if err != nil { + return fmt.Errorf("%w: bad uncles: %v", errInvalidBody, err) + } + uncleLists = append(uncleLists, uncles) + if bodies[index].Withdrawals != nil { + withdrawals, err := bodies[index].Withdrawals.Items() + if err != nil { + return fmt.Errorf("%w: bad withdrawals: %v", errInvalidBody, err) } + withdrawalLists = append(withdrawalLists, withdrawals) } else { - if blobs != 0 { - return errInvalidBody - } + withdrawalLists = append(withdrawalLists, nil) } return nil } @@ -628,8 +616,9 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH result.Withdrawals = withdrawalLists[index] result.SetBodyDone() } + nresults := len(hashes.TransactionRoots) return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, - bodyReqTimer, bodyInMeter, bodyDropMeter, len(txLists), validate, reconstruct) + bodyReqTimer, bodyInMeter, bodyDropMeter, nresults, validate, reconstruct) } // DeliverReceipts injects a receipt retrieval response into the results queue. diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index c990e5b950..97584341ec 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -30,8 +30,10 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -323,26 +325,31 @@ func XTestDelivery(t *testing.T) { emptyList []*types.Header txset [][]*types.Transaction uncleset [][]*types.Header + bodies []eth.BlockBody ) numToSkip := rand.Intn(len(f.Headers)) for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { - txset = append(txset, world.getTransactions(hdr.Number.Uint64())) + txs := world.getTransactions(hdr.Number.Uint64()) + txset = append(txset, txs) uncleset = append(uncleset, emptyList) + txsList, _ := rlp.EncodeToRawList(txs) + bodies = append(bodies, eth.BlockBody{Transactions: txsList}) + } + hashes := eth.BlockBodyHashes{ + TransactionRoots: make([]common.Hash, len(txset)), + UncleHashes: make([]common.Hash, len(uncleset)), + WithdrawalRoots: make([]common.Hash, len(txset)), } - var ( - txsHashes = make([]common.Hash, len(txset)) - uncleHashes = make([]common.Hash, len(uncleset)) - ) hasher := trie.NewStackTrie(nil) for i, txs := range txset { - txsHashes[i] = types.DeriveSha(types.Transactions(txs), hasher) + hashes.TransactionRoots[i] = types.DeriveSha(types.Transactions(txs), hasher) } for i, uncles := range uncleset { - uncleHashes[i] = types.CalcUncleHash(uncles) + hashes.UncleHashes[i] = types.CalcUncleHash(uncles) } + time.Sleep(100 * time.Millisecond) - _, err := q.DeliverBodies(peer.id, txset, txsHashes, uncleset, uncleHashes, nil, nil) - if err != nil { + if _, err := q.DeliverBodies(peer.id, hashes, bodies); err != nil { fmt.Printf("delivered %d bodies %v\n", len(txset), err) } } else { diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index 2cf9c4672b..e693bfc066 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -64,6 +64,12 @@ var errSyncMerged = errors.New("sync merged") // should abort and restart with the new state. var errSyncReorged = errors.New("sync reorged") +// errSyncTrimmed is an internal helper error to signal that the local chain +// has been trimmed (e.g, via debug_setHead explicitly) and the skeleton chain +// is no longer linked with the local chain. In this case, the skeleton sync +// should be re-scheduled again. +var errSyncTrimmed = errors.New("sync trimmed") + // errTerminated is returned if the sync mechanism was terminated for this run of // the process. This is usually the case when Geth is shutting down and some events // might still be propagating. @@ -201,6 +207,7 @@ type backfiller interface { type skeleton struct { db ethdb.Database // Database backing the skeleton filler backfiller // Chain syncer suspended/resumed by head events + chain chainReader // Underlying block chain peers *peerSet // Set of peers we can sync from idles map[string]*peerConnection // Set of idle peers in the current sync cycle @@ -225,12 +232,19 @@ type skeleton struct { syncStarting func() // callback triggered after a sync cycle is inited but before started } +// chainReader wraps the method to retrieve the head of the local chain. +type chainReader interface { + // CurrentSnapBlock retrieves the head snap block from the local chain. + CurrentSnapBlock() *types.Header +} + // newSkeleton creates a new sync skeleton that tracks a potentially dangling // header chain until it's linked into an existing set of blocks. -func newSkeleton(db ethdb.Database, peers *peerSet, drop peerDropFn, filler backfiller) *skeleton { +func newSkeleton(db ethdb.Database, peers *peerSet, drop peerDropFn, filler backfiller, chain chainReader) *skeleton { sk := &skeleton{ db: db, filler: filler, + chain: chain, peers: peers, drop: drop, requests: make(map[uint64]*headerRequest), @@ -296,6 +310,11 @@ func (s *skeleton) startup() { // head to force a cleanup. head = newhead + case err == errSyncTrimmed: + // The skeleton chain is not linked with the local chain anymore, + // restart the sync. + head = nil + case err == errTerminated: // Sync was requested to be terminated from within, stop and // return (no need to pass a message, was already done internally) @@ -343,6 +362,29 @@ func (s *skeleton) Sync(head *types.Header, final *types.Header, force bool) err } } +// linked returns the flag indicating whether the skeleton has been linked with +// the local chain. +func (s *skeleton) linked(number uint64, hash common.Hash) bool { + linked := rawdb.HasHeader(s.db, hash, number) && + rawdb.HasBody(s.db, hash, number) && + rawdb.HasReceipts(s.db, hash, number) + + // Ensure the skeleton chain links to the local chain below the chain head. + // This accounts for edge cases where leftover chain segments above the head + // may still link to the skeleton chain. In such cases, synchronization is + // likely to fail due to potentially missing segments in the middle. + // + // You can try to produce the edge case by these steps: + // - sync the chain + // - debug.setHead(`0x1`) + // - kill the geth process (the chain segment will be left with chain head rewound) + // - restart + if s.chain.CurrentSnapBlock() != nil { + linked = linked && s.chain.CurrentSnapBlock().Number.Uint64() >= number + } + return linked +} + // sync is the internal version of Sync that executes a single sync cycle, either // until some termination condition is reached, or until the current cycle merges // with a previously aborted run. @@ -367,10 +409,7 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // If the sync is already done, resume the backfiller. When the loop stops, // terminate the backfiller too. - linked := len(s.progress.Subchains) == 1 && - rawdb.HasHeader(s.db, s.progress.Subchains[0].Next, s.scratchHead) && - rawdb.HasBody(s.db, s.progress.Subchains[0].Next, s.scratchHead) && - rawdb.HasReceipts(s.db, s.progress.Subchains[0].Next, s.scratchHead) + linked := len(s.progress.Subchains) == 1 && s.linked(s.scratchHead, s.progress.Subchains[0].Next) if linked { s.filler.resume() } @@ -486,7 +525,17 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // is still running, it will pick it up. If it already terminated, // a new cycle needs to be spun up. if linked { - s.filler.resume() + if len(s.progress.Subchains) == 1 && s.linked(s.scratchHead, s.progress.Subchains[0].Next) { + // The skeleton chain has been extended and is still linked with the local + // chain, try to re-schedule the backfiller if it's already terminated. + s.filler.resume() + } else { + // The skeleton chain is no longer linked to the local chain for some reason + // (e.g. debug_setHead was used to trim the local chain). Re-schedule the + // skeleton sync to fill the chain gap. + log.Warn("Local chain has been trimmed", "tailnumber", s.scratchHead, "tailhash", s.progress.Subchains[0].Next) + return nil, errSyncTrimmed + } } case req := <-requestFails: @@ -649,9 +698,19 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error // Not a noop / double head announce, abort with a reorg return fmt.Errorf("%w, tail: %d, head: %d, newHead: %d", errChainReorged, lastchain.Tail, lastchain.Head, number) } + // Terminate the sync if the chain head is gapped if lastchain.Head+1 < number { return fmt.Errorf("%w, head: %d, newHead: %d", errChainGapped, lastchain.Head, number) } + // Ignore the duplicated beacon header announcement + if lastchain.Head == number { + local := rawdb.ReadSkeletonHeader(s.db, number) + if local != nil && local.Hash() == head.Hash() { + log.Debug("Ignored the identical beacon header", "number", number, "hash", local.Hash()) + return nil + } + } + // Terminate the sync if the chain head is forked if parent := rawdb.ReadSkeletonHeader(s.db, number-1); parent.Hash() != head.ParentHash { return fmt.Errorf("%w, ancestor: %d, hash: %s, want: %s", errChainForked, number-1, parent.Hash(), head.ParentHash) } @@ -669,6 +728,7 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error if err := batch.Write(); err != nil { log.Crit("Failed to write skeleton sync status", "err", err) } + log.Debug("Extended beacon header chain", "number", head.Number, "hash", head.Hash()) return nil } @@ -909,6 +969,45 @@ func (s *skeleton) revertRequest(req *headerRequest) { s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } +// mergeSubchains is invoked once certain beacon headers have been persisted locally +// and the subchains should be merged in case there are some overlaps between. An +// indicator will be returned if the last subchain is merged with previous subchain. +func (s *skeleton) mergeSubchains() bool { + // If the subchain extended into the next subchain, we need to handle + // the overlap. Since there could be many overlaps, do this in a loop. + var merged bool + for len(s.progress.Subchains) > 1 && s.progress.Subchains[1].Head >= s.progress.Subchains[0].Tail { + // Extract some stats from the second subchain + head := s.progress.Subchains[1].Head + tail := s.progress.Subchains[1].Tail + next := s.progress.Subchains[1].Next + + // Since we just overwrote part of the next subchain, we need to trim + // its head independent of matching or mismatching content + if s.progress.Subchains[1].Tail >= s.progress.Subchains[0].Tail { + // Fully overwritten, get rid of the subchain as a whole + log.Debug("Previous subchain fully overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + continue + } else { + // Partially overwritten, trim the head to the overwritten size + log.Debug("Previous subchain partially overwritten", "head", head, "tail", tail, "next", next) + s.progress.Subchains[1].Head = s.progress.Subchains[0].Tail - 1 + } + // If the old subchain is an extension of the new one, merge the two + // and let the skeleton syncer restart (to clean internal state) + if rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[1].Head).Hash() == s.progress.Subchains[0].Next { + log.Debug("Previous subchain merged", "head", head, "tail", tail, "next", next) + s.progress.Subchains[0].Tail = s.progress.Subchains[1].Tail + s.progress.Subchains[0].Next = s.progress.Subchains[1].Next + + s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) + merged = true + } + } + return merged +} + func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged bool) { res.peer.log.Trace("Processing header response", "head", res.headers[0].Number, "hash", res.headers[0].Hash(), "count", len(res.headers)) @@ -982,10 +1081,9 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // processing is done, so it's just one more "needless" check. // // The weird cascading checks are done to minimize the database reads. - linked = rawdb.HasHeader(s.db, header.ParentHash, header.Number.Uint64()-1) && - rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) && - rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) + linked = s.linked(header.Number.Uint64()-1, header.ParentHash) if linked { + log.Debug("Primary subchain linked", "number", header.Number.Uint64()-1, "hash", header.ParentHash) break } } @@ -999,6 +1097,9 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // If the beacon chain was linked to the local chain, completely swap out // all internal progress and abort header synchronization. if linked { + // Merge all overlapped subchains beforehand + s.mergeSubchains() + // Linking into the local chain should also mean that there are no // leftover subchains, but in the case of importing the blocks via // the engine API, we will not push the subchains forward. This will @@ -1056,41 +1157,10 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo s.scratchHead -= uint64(consumed) - // If the subchain extended into the next subchain, we need to handle - // the overlap. Since there could be many overlaps (come on), do this - // in a loop. - for len(s.progress.Subchains) > 1 && s.progress.Subchains[1].Head >= s.progress.Subchains[0].Tail { - // Extract some stats from the second subchain - head := s.progress.Subchains[1].Head - tail := s.progress.Subchains[1].Tail - next := s.progress.Subchains[1].Next - - // Since we just overwrote part of the next subchain, we need to trim - // its head independent of matching or mismatching content - if s.progress.Subchains[1].Tail >= s.progress.Subchains[0].Tail { - // Fully overwritten, get rid of the subchain as a whole - log.Debug("Previous subchain fully overwritten", "head", head, "tail", tail, "next", next) - s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) - continue - } else { - // Partially overwritten, trim the head to the overwritten size - log.Debug("Previous subchain partially overwritten", "head", head, "tail", tail, "next", next) - s.progress.Subchains[1].Head = s.progress.Subchains[0].Tail - 1 - } - // If the old subchain is an extension of the new one, merge the two - // and let the skeleton syncer restart (to clean internal state) - if rawdb.ReadSkeletonHeader(s.db, s.progress.Subchains[1].Head).Hash() == s.progress.Subchains[0].Next { - log.Debug("Previous subchain merged", "head", head, "tail", tail, "next", next) - s.progress.Subchains[0].Tail = s.progress.Subchains[1].Tail - s.progress.Subchains[0].Next = s.progress.Subchains[1].Next - - s.progress.Subchains = append(s.progress.Subchains[:1], s.progress.Subchains[2:]...) - merged = true - } - } // If subchains were merged, all further available headers in the scratch // space are invalid since we skipped ahead. Stop processing the scratch // space to avoid dropping peers thinking they delivered invalid data. + merged = s.mergeSubchains() if merged { break } @@ -1121,15 +1191,17 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // due to the downloader backfilling past the tracked tail. func (s *skeleton) cleanStales(filled *types.Header) error { number := filled.Number.Uint64() - log.Trace("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) + log.Debug("Cleaning stale beacon headers", "filled", number, "hash", filled.Hash()) - // If the filled header is below the linked subchain, something's corrupted - // internally. Report and error and refuse to do anything. + // If the filled header is below the subchain, it means the skeleton is not + // linked with local chain yet, don't bother to do cleanup. if number+1 < s.progress.Subchains[0].Tail { - return fmt.Errorf("filled header below beacon header tail: %d < %d", number, s.progress.Subchains[0].Tail) + log.Debug("filled header below beacon header tail", "filled", number, "tail", s.progress.Subchains[0].Tail) + return nil } // If nothing in subchain is filled, don't bother to do cleanup. if number+1 == s.progress.Subchains[0].Tail { + log.Debug("Skeleton chain not yet consumed", "filled", number, "hash", filled.Hash(), "tail", s.progress.Subchains[0].Tail) return nil } // If the latest fill was on a different subchain, it means the backfiller @@ -1206,6 +1278,7 @@ func (s *skeleton) cleanStales(filled *types.Header) error { if err := batch.Write(); err != nil { log.Crit("Failed to write beacon trim data", "err", err) } + log.Debug("Cleaned stale beacon headers", "start", start, "end", end) return nil } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 4aa97cf1f7..5c54b4b5c2 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "sync/atomic" "testing" @@ -71,6 +72,12 @@ func (hf *hookedBackfiller) resume() { } } +type fakeChainReader struct{} + +func (fc *fakeChainReader) CurrentSnapBlock() *types.Header { + return &types.Header{Number: big.NewInt(math.MaxInt64)} +} + // skeletonTestPeer is a mock peer that can only serve header requests from a // pre-perated header chain (which may be arbitrarily wrong for testing). // @@ -369,7 +376,7 @@ func TestSkeletonSyncInit(t *testing.T) { // Create a skeleton sync and run a cycle wait := make(chan struct{}) - skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller(), &fakeChainReader{}) skeleton.syncStarting = func() { close(wait) } skeleton.Sync(tt.head, nil, true) @@ -472,7 +479,7 @@ func TestSkeletonSyncExtend(t *testing.T) { // Create a skeleton sync and run a cycle wait := make(chan struct{}) - skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller()) + skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller(), &fakeChainReader{}) skeleton.syncStarting = func() { close(wait) } skeleton.Sync(tt.head, nil, true) @@ -885,7 +892,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { } } // Create a skeleton sync and run a cycle - skeleton := newSkeleton(db, peerset, drop, filler) + skeleton := newSkeleton(db, peerset, drop, filler, &fakeChainReader{}) skeleton.Sync(tt.head, nil, true) // Wait a bit (bleah) for the initial sync loop to go to idle. This might diff --git a/eth/downloader/syncmode.go b/eth/downloader/syncmode.go new file mode 100644 index 0000000000..036119ce3d --- /dev/null +++ b/eth/downloader/syncmode.go @@ -0,0 +1,115 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "sync" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// syncModer is responsible for managing the downloader's sync mode. It takes the +// user's preference at startup and then determines the appropriate sync mode +// based on the current chain status. +type syncModer struct { + mode ethconfig.SyncMode + chain BlockChain + disk ethdb.KeyValueReader + lock sync.Mutex +} + +func newSyncModer(mode ethconfig.SyncMode, chain BlockChain, disk ethdb.KeyValueReader) *syncModer { + if mode == ethconfig.FullSync { + // The database seems empty as the current block is the genesis. Yet the snap + // block is ahead, so snap sync was enabled for this node at a certain point. + // The scenarios where this can happen is + // * if the user manually (or via a bad block) rolled back a snap sync node + // below the sync point. + // * the last snap sync is not finished while user specifies a full sync this + // time. But we don't have any recent state for full sync. + // In these cases however it's safe to reenable snap sync. + fullBlock, snapBlock := chain.CurrentBlock(), chain.CurrentSnapBlock() + if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 { + mode = ethconfig.SnapSync + log.Warn("Switching from full-sync to snap-sync", "reason", "snap-sync incomplete") + } else if !chain.HasState(fullBlock.Root) { + mode = ethconfig.SnapSync + log.Warn("Switching from full-sync to snap-sync", "reason", "head state missing") + } else { + // Grant the full sync mode + log.Info("Enabled full-sync", "head", fullBlock.Number, "hash", fullBlock.Hash()) + } + } else { + head := chain.CurrentBlock() + if head.Number.Uint64() > 0 && chain.HasState(head.Root) { + mode = ethconfig.FullSync + log.Info("Switching from snap-sync to full-sync", "reason", "snap-sync complete") + } else { + // If snap sync was requested and our database is empty, grant it + log.Info("Enabled snap-sync", "head", head.Number, "hash", head.Hash()) + } + } + return &syncModer{ + mode: mode, + chain: chain, + disk: disk, + } +} + +// get retrieves the current sync mode, either explicitly set, or derived +// from the chain status. +func (m *syncModer) get(report bool) ethconfig.SyncMode { + m.lock.Lock() + defer m.lock.Unlock() + + // If we're in snap sync mode, return that directly + if m.mode == ethconfig.SnapSync { + return ethconfig.SnapSync + } + logger := log.Debug + if report { + logger = log.Info + } + // We are probably in full sync, but we might have rewound to before the + // snap sync pivot, check if we should re-enable snap sync. + head := m.chain.CurrentBlock() + if pivot := rawdb.ReadLastPivotNumber(m.disk); pivot != nil { + if head.Number.Uint64() < *pivot { + logger("Reenabled snap-sync as chain is lagging behind the pivot", "head", head.Number, "pivot", pivot) + return ethconfig.SnapSync + } + } + // We are in a full sync, but the associated head state is missing. To complete + // the head state, forcefully rerun the snap sync. Note it doesn't mean the + // persistent state is corrupted, just mismatch with the head block. + if !m.chain.HasState(head.Root) { + logger("Reenabled snap-sync as chain is stateless") + return ethconfig.SnapSync + } + // Nope, we're really full syncing + return ethconfig.FullSync +} + +// disableSnap disables the snap sync mode, usually it's called after a successful snap sync. +func (m *syncModer) disableSnap() { + m.lock.Lock() + m.mode = ethconfig.FullSync + m.lock.Unlock() +} diff --git a/eth/dropper.go b/eth/dropper.go index 51f2a7a95a..dada5d07c0 100644 --- a/eth/dropper.go +++ b/eth/dropper.go @@ -145,6 +145,9 @@ func randomDuration(min, max time.Duration) time.Duration { if min > max { panic("min duration must be less than or equal to max duration") } + if min == max { + return min + } return time.Duration(mrand.Int63n(int64(max-min)) + int64(min)) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7b8cd78fa6..765bd59b34 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -50,30 +50,34 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: history.KeepAll, - SyncMode: SnapSync, - NetworkId: 0, // enable auto configuration of networkID == chainID - TxLookupLimit: 2350000, - TransactionHistory: 2350000, - LogHistory: 2350000, - StateHistory: params.FullImmutabilityThreshold, - MaxDiffLayers: pathdb.DefaultMaxDiffLayers, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, - FilterLogCacheSize: 32, - LogQueryLimit: 1000, - Miner: miner.DefaultConfig, - TxPool: legacypool.DefaultConfig, - BlobPool: blobpool.DefaultConfig, - RPCGasCap: 50000000, - RPCEVMTimeout: 5 * time.Second, - GPO: FullNodeGPO, - RPCTxFeeCap: 1, // 1 ether - TxSyncDefaultTimeout: 20 * time.Second, - TxSyncMaxTimeout: 1 * time.Minute, + HistoryMode: history.KeepAll, + SyncMode: SnapSync, + NetworkId: 0, // enable auto configuration of networkID == chainID + TxLookupLimit: 2350000, + TransactionHistory: 2350000, + LogHistory: 2350000, + StateHistory: pathdb.Defaults.StateHistory, + MaxDiffLayers: pathdb.DefaultMaxDiffLayers, + TrienodeHistory: pathdb.Defaults.TrienodeHistory, + NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, + DatabaseCache: 512, + TrieCleanCache: 154, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 102, + FilterLogCacheSize: 32, + LogQueryLimit: 1000, + Miner: miner.DefaultConfig, + TxPool: legacypool.DefaultConfig, + BlobPool: blobpool.DefaultConfig, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + GPO: FullNodeGPO, + RPCTxFeeCap: 1, // 1 ether + TxSyncDefaultTimeout: 20 * time.Second, + TxSyncMaxTimeout: 1 * time.Minute, + SlowBlockThreshold: -1, // Disabled by default; set via --debug.logslowblock flag + RangeLimit: 0, } //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go @@ -110,6 +114,13 @@ type Config struct { LogExportCheckpoints string // export log index checkpoints to file StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. MaxDiffLayers int `toml:",omitempty"` // Maximum diff layers allowed in the layer tree. + TrienodeHistory int64 `toml:",omitempty"` // Number of blocks from the chain head for which trienode histories are retained + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 `toml:",omitempty"` // State scheme represents the scheme used to store ethereum states and trie // nodes on top. It can be 'hash', 'path', or none which means use the scheme @@ -121,6 +132,11 @@ type Config struct { // presence of these blocks for every new peer connection. RequiredBlocks map[uint64]common.Hash `toml:"-"` + // SlowBlockThreshold is the block execution time threshold beyond which + // detailed statistics are logged. Negative means disabled (default), zero + // logs all blocks, positive filters by execution time. + SlowBlockThreshold time.Duration `toml:",omitempty"` + // Database options SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` @@ -192,6 +208,9 @@ type Config struct { // EIP-7966: eth_sendRawTransactionSync timeouts TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + + // RangeLimit restricts the maximum range (end - start) for range queries. + RangeLimit uint64 `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 6f18dc34c5..6f94a409e5 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -31,8 +31,11 @@ func (c Config) MarshalTOML() (interface{}, error) { LogNoHistory bool `toml:",omitempty"` LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` + TrienodeHistory int64 `toml:",omitempty"` + NodeFullValueCheckpoint uint32 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` + SlowBlockThreshold time.Duration `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int @@ -64,6 +67,7 @@ func (c Config) MarshalTOML() (interface{}, error) { OverrideVerkle *uint64 `toml:",omitempty"` TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + RangeLimit uint64 `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -80,8 +84,11 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LogNoHistory = c.LogNoHistory enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory + enc.TrienodeHistory = c.TrienodeHistory + enc.NodeFullValueCheckpoint = c.NodeFullValueCheckpoint enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks + enc.SlowBlockThreshold = c.SlowBlockThreshold enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -113,6 +120,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.OverrideVerkle = c.OverrideVerkle enc.TxSyncDefaultTimeout = c.TxSyncDefaultTimeout enc.TxSyncMaxTimeout = c.TxSyncMaxTimeout + enc.RangeLimit = c.RangeLimit return &enc, nil } @@ -133,8 +141,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LogNoHistory *bool `toml:",omitempty"` LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` + TrienodeHistory *int64 `toml:",omitempty"` + NodeFullValueCheckpoint *uint32 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` + SlowBlockThreshold *time.Duration `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int @@ -166,6 +177,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { OverrideVerkle *uint64 `toml:",omitempty"` TxSyncDefaultTimeout *time.Duration `toml:",omitempty"` TxSyncMaxTimeout *time.Duration `toml:",omitempty"` + RangeLimit *uint64 `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -213,12 +225,21 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } + if dec.TrienodeHistory != nil { + c.TrienodeHistory = *dec.TrienodeHistory + } + if dec.NodeFullValueCheckpoint != nil { + c.NodeFullValueCheckpoint = *dec.NodeFullValueCheckpoint + } if dec.StateScheme != nil { c.StateScheme = *dec.StateScheme } if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } + if dec.SlowBlockThreshold != nil { + c.SlowBlockThreshold = *dec.SlowBlockThreshold + } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } @@ -312,5 +333,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxSyncMaxTimeout != nil { c.TxSyncMaxTimeout = *dec.TxSyncMaxTimeout } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } return nil } diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 4954c71819..50d6f2f7ad 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -171,10 +171,10 @@ type TxFetcher struct { alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails // Callbacks - hasTx func(common.Hash) bool // Retrieves a tx from the local txpool - addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool - fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer - dropPeer func(string) // Drops a peer in case of announcement violation + validateMeta func(common.Hash, byte) error // Validate a tx metadata based on the local txpool + addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool + fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer + dropPeer func(string) // Drops a peer in case of announcement violation step chan struct{} // Notification channel when the fetcher loop iterates clock mclock.Clock // Monotonic clock or simulated clock for tests @@ -184,36 +184,36 @@ type TxFetcher struct { // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { - return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) +func NewTxFetcher(validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { + return NewTxFetcherForTests(validateMeta, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. func NewTxFetcherForTests( - hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), + validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ - notify: make(chan *txAnnounce), - cleanup: make(chan *txDelivery), - drop: make(chan *txDrop), - quit: make(chan struct{}), - waitlist: make(map[common.Hash]map[string]struct{}), - waittime: make(map[common.Hash]mclock.AbsTime), - waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), - announces: make(map[string]map[common.Hash]*txMetadataWithSeq), - announced: make(map[common.Hash]map[string]struct{}), - fetching: make(map[common.Hash]string), - requests: make(map[string]*txRequest), - alternates: make(map[common.Hash]map[string]struct{}), - underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), - hasTx: hasTx, - addTxs: addTxs, - fetchTxs: fetchTxs, - dropPeer: dropPeer, - clock: clock, - realTime: realTime, - rand: rand, + notify: make(chan *txAnnounce), + cleanup: make(chan *txDelivery), + drop: make(chan *txDrop), + quit: make(chan struct{}), + waitlist: make(map[common.Hash]map[string]struct{}), + waittime: make(map[common.Hash]mclock.AbsTime), + waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), + announces: make(map[string]map[common.Hash]*txMetadataWithSeq), + announced: make(map[common.Hash]map[string]struct{}), + fetching: make(map[common.Hash]string), + requests: make(map[string]*txRequest), + alternates: make(map[common.Hash]map[string]struct{}), + underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), + validateMeta: validateMeta, + addTxs: addTxs, + fetchTxs: fetchTxs, + dropPeer: dropPeer, + clock: clock, + realTime: realTime, + rand: rand, } } @@ -236,19 +236,26 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c underpriced int64 ) for i, hash := range hashes { - switch { - case f.hasTx(hash): + err := f.validateMeta(hash, types[i]) + if errors.Is(err, txpool.ErrAlreadyKnown) { duplicate++ - case f.isKnownUnderpriced(hash): - underpriced++ - default: - unknownHashes = append(unknownHashes, hash) + continue + } + if err != nil { + continue + } - // Transaction metadata has been available since eth68, and all - // legacy eth protocols (prior to eth68) have been deprecated. - // Therefore, metadata is always expected in the announcement. - unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) + if f.isKnownUnderpriced(hash) { + underpriced++ + continue } + + unknownHashes = append(unknownHashes, hash) + + // Transaction metadata has been available since eth68, and all + // legacy eth protocols (prior to eth68) have been deprecated. + // Therefore, metadata is always expected in the announcement. + unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) } txAnnounceKnownMeter.Mark(duplicate) txAnnounceUnderpricedMeter.Mark(underpriced) @@ -358,9 +365,9 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) otherRejectMeter.Mark(otherreject) // If 'other reject' is >25% of the deliveries in any batch, sleep a bit. - if otherreject > addTxsBatchSize/4 { + if otherreject > int64((len(batch)+3)/4) { + log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject) time.Sleep(200 * time.Millisecond) - log.Debug("Peer delivering stale transactions", "peer", peer, "rejected", otherreject) } // If we encountered a protocol violation, disconnect this peer. if violation != nil { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index e685d50c5c..87fbe9f38c 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -87,6 +87,19 @@ type txFetcherTest struct { steps []interface{} } +// newTestTxFetcher creates a tx fetcher with noop callbacks, simulated clock, +// and deterministic randomness. +func newTestTxFetcher() *TxFetcher { + return NewTxFetcher( + func(common.Hash, byte) error { return nil }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + nil, + ) +} + // Tests that transaction announcements with associated metadata are added to a // waitlist, and none of them are scheduled for retrieval until the wait expires. // @@ -95,14 +108,7 @@ type txFetcherTest struct { // with all the useless extra fields. func TestTransactionFetcherWaiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -297,14 +303,7 @@ func TestTransactionFetcherWaiting(t *testing.T) { // already scheduled. func TestTransactionFetcherSkipWaiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{ @@ -387,14 +386,7 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { // and subsequent announces block or get allotted to someone else. func TestTransactionFetcherSingletonRequesting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -493,15 +485,12 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { proceed := make(chan struct{}) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(origin string, hashes []common.Hash) error { - <-proceed - return errors.New("peer disconnected") - }, - nil, - ) + f := newTestTxFetcher() + f.fetchTxs = func(origin string, hashes []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + } + return f }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage @@ -576,16 +565,7 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { // are cleaned up. func TestTransactionFetcherCleanup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -620,16 +600,7 @@ func TestTransactionFetcherCleanup(t *testing.T) { // this was a bug)). func TestTransactionFetcherCleanupEmpty(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -663,16 +634,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { // different peer, or self if they are after the cutoff point. func TestTransactionFetcherMissingRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", @@ -724,16 +686,7 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { // delivered, the peer gets properly cleaned out from the internal state. func TestTransactionFetcherMissingCleanup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{peer: "A", @@ -773,16 +726,7 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { // Tests that transaction broadcasts properly clean up announcements. func TestTransactionFetcherBroadcasts(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up three transactions to be in different stats, waiting, queued and fetching doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -829,14 +773,7 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { // Tests that the waiting list timers properly reset and reschedule. func TestTransactionFetcherWaitTimerResets(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, isWaiting(map[string][]announce{ @@ -899,16 +836,7 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { // out and be re-scheduled for someone else. func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Push an initial announcement through to the scheduled stage doTxNotify{ @@ -977,14 +905,7 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { // Tests that the fetching timeout timers properly reset and reschedule. func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, @@ -1055,14 +976,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { }) } testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce all the transactions, wait a bit and ensure only a small // percentage gets requested @@ -1085,14 +999,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { // be requested at a time, to keep the responses below a reasonable level. func TestTransactionFetcherBandwidthLimiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce mid size transactions from A to verify that multiple // ones can be piled into a single request. @@ -1202,14 +1109,7 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { }) } testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Announce half of the transaction and wait for them to be scheduled doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2], types: typesA[:maxTxAnnounces/2], sizes: sizesA[:maxTxAnnounces/2]}, @@ -1270,24 +1170,21 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { func TestTransactionFetcherUnderpricedDedup(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - errs := make([]error, len(txs)) - for i := 0; i < len(errs); i++ { - if i%3 == 0 { - errs[i] = txpool.ErrUnderpriced - } else if i%3 == 1 { - errs[i] = txpool.ErrReplaceUnderpriced - } else { - errs[i] = txpool.ErrTxGasPriceTooLow - } + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + if i%3 == 0 { + errs[i] = txpool.ErrUnderpriced + } else if i%3 == 1 { + errs[i] = txpool.ErrReplaceUnderpriced + } else { + errs[i] = txpool.ErrTxGasPriceTooLow } - return errs - }, - func(string, []common.Hash) error { return nil }, - nil, - ) + } + return errs + } + return f }, steps: []interface{}{ // Deliver a transaction through the fetcher, but reject as underpriced @@ -1371,18 +1268,15 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { } testTransactionFetcher(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - errs := make([]error, len(txs)) - for i := 0; i < len(errs); i++ { - errs[i] = txpool.ErrUnderpriced - } - return errs - }, - func(string, []common.Hash) error { return nil }, - nil, - ) + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + errs[i] = txpool.ErrUnderpriced + } + return errs + } + return f }, steps: append(steps, []interface{}{ // The preparation of the test has already been done in `steps`, add the last check @@ -1402,16 +1296,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { // Tests that unexpected deliveries don't corrupt the internal state. func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Deliver something out of the blue isWaiting(nil), @@ -1461,16 +1346,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { // live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up a few hashes into various stages doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, @@ -1535,16 +1411,7 @@ func TestTransactionFetcherDrop(t *testing.T) { // available peer. func TestTransactionFetcherDropRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Set up a few hashes into various stages doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, @@ -1582,14 +1449,9 @@ func TestInvalidAnnounceMetadata(t *testing.T) { drop := make(chan string, 2) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - func(peer string) { drop <- peer }, - ) + f := newTestTxFetcher() + f.dropPeer = func(peer string) { drop <- peer } + return f }, steps: []interface{}{ // Initial announcement to get something into the waitlist @@ -1664,16 +1526,7 @@ func TestInvalidAnnounceMetadata(t *testing.T) { // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -1692,16 +1545,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, @@ -1722,16 +1566,7 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { // with a concurrent notify. func TestTransactionFetcherFuzzCrash03(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast doTxNotify{ @@ -1762,17 +1597,12 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { - <-proceed - return errors.New("peer disconnected") - }, - nil, - ) + f := newTestTxFetcher() + f.fetchTxs = func(string, []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + } + return f }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast @@ -1796,14 +1626,7 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { // once they are announced in the network. func TestBlobTransactionAnnounce(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, @@ -1864,16 +1687,7 @@ func TestBlobTransactionAnnounce(t *testing.T) { func TestTransactionFetcherDropAlternates(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - return make([]error, len(txs)) - }, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, + init: newTestTxFetcher, steps: []interface{}{ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, @@ -1912,6 +1726,30 @@ func TestTransactionFetcherDropAlternates(t *testing.T) { }) } +func TestTransactionFetcherWrongMetadata(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + f := newTestTxFetcher() + f.validateMeta = func(name common.Hash, kind byte) error { + switch kind { + case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: + return nil + } + return types.ErrTxTypeNotSupported + } + return f + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{0xff, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + }), + }, + }) +} + func makeInvalidBlobTx() *types.Transaction { key, _ := crypto.GenerateKey() blob := &kzg4844.Blob{byte(0xa)} @@ -1947,20 +1785,16 @@ func TestTransactionProtocolViolation(t *testing.T) { ) testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { - var errs []error - for range txs { - errs = append(errs, txpool.ErrKZGVerificationError) - } - return errs - }, - func(a string, b []common.Hash) error { - return nil - }, - func(peer string) { drop <- struct{}{} }, - ) + f := newTestTxFetcher() + f.addTxs = func(txs []*types.Transaction) []error { + var errs []error + for range txs { + errs = append(errs, txpool.ErrKZGVerificationError) + } + return errs + } + f.dropPeer = func(string) { drop <- struct{}{} } + return f }, steps: []interface{}{ // Initial announcement to get something into the waitlist @@ -2357,7 +2191,7 @@ func TestTransactionForgotten(t *testing.T) { } fetcher := NewTxFetcherForTests( - func(common.Hash) bool { return false }, + func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { errs := make([]error, len(txs)) for i := 0; i < len(errs); i++ { diff --git a/eth/filters/api.go b/eth/filters/api.go index 9b418bd151..edda4dae5e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,17 +36,29 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errInvalidBlockRange = errors.New("invalid block range params") + errInvalidTopic = invalidParamsErr("invalid topic(s)") + errInvalidBlockRange = invalidParamsErr("invalid block range params") + errBlockRangeIntoFuture = invalidParamsErr("block range extends beyond current head block") + errBlockHashWithRange = invalidParamsErr("can't specify fromBlock/toBlock with blockHash") + errPendingLogsUnsupported = invalidParamsErr("pending logs are not supported") errUnknownBlock = errors.New("unknown block") - errBlockHashWithRange = errors.New("can't specify fromBlock/toBlock with blockHash") - errPendingLogsUnsupported = errors.New("pending logs are not supported") + errFilterNotFound = errors.New("filter not found") errExceedMaxTopics = errors.New("exceed max topics") errExceedLogQueryLimit = errors.New("exceed max addresses or topics per search position") errExceedMaxTxHashes = errors.New("exceed max number of transaction hashes allowed per transactionReceipts subscription") ) +type invalidParamsError struct { + err error +} + +func (e invalidParamsError) Error() string { return e.err.Error() } +func (e invalidParamsError) ErrorCode() int { return -32602 } + +func invalidParamsErr(format string, args ...any) error { + return invalidParamsError{fmt.Errorf(format, args...)} +} + const ( // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 maxTopics = 4 @@ -78,6 +90,7 @@ type FilterAPI struct { filters map[rpc.ID]*filter timeout time.Duration logQueryLimit int + rangeLimit uint64 } // NewFilterAPI returns a new FilterAPI instance. @@ -88,6 +101,7 @@ func NewFilterAPI(system *FilterSystem) *FilterAPI { filters: make(map[rpc.ID]*filter), timeout: system.cfg.Timeout, logQueryLimit: system.cfg.LogQueryLimit, + rangeLimit: system.cfg.RangeLimit, } go api.timeoutLoop(system.cfg.Timeout) @@ -479,7 +493,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type return nil, &history.PrunedHistoryError{} } // Construct the range filter - filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics, api.rangeLimit) } // Run the filter and return all the logs @@ -531,7 +545,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo end = f.crit.ToBlock.Int64() } // Construct the range filter - filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics, api.rangeLimit) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index ae7cd1c298..fcdd0f9186 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -19,6 +19,7 @@ package filters import ( "context" "errors" + "fmt" "math" "time" @@ -42,15 +43,17 @@ type Filter struct { begin, end int64 // Range interval if filtering multiple blocks rangeLogsTestHook chan rangeLogsTestEvent + rangeLimit uint64 } // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash, rangeLimit uint64) *Filter { // Create a generic filter and convert it into a range filter filter := newFilter(sys, addresses, topics) filter.begin = begin filter.end = end + filter.rangeLimit = rangeLimit return filter } @@ -141,6 +144,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { if err != nil { return nil, err } + if f.rangeLimit != 0 && (end-begin) > f.rangeLimit { + return nil, fmt.Errorf("exceed maximum block range: %d", f.rangeLimit) + } return f.rangeLogs(ctx, begin, end) } @@ -219,9 +225,12 @@ func (s *searchSession) updateChainView() error { if lastBlock == math.MaxUint64 { lastBlock = head } - if firstBlock > lastBlock || lastBlock > head { + if firstBlock > lastBlock { return errInvalidBlockRange } + if lastBlock > head { + return errBlockRangeIntoFuture + } s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) // Trim existing match set in case a reorg may have invalidated some results @@ -526,7 +535,7 @@ type ReceiptWithTx struct { // In addition to returning receipts, it also returns the corresponding transactions. // This is because receipts only contain low-level data, while user-facing data // may require additional information from the Transaction. -func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*ReceiptWithTx { +func filterReceipts(txHashes map[common.Hash]struct{}, ev core.ChainEvent) []*ReceiptWithTx { var ret []*ReceiptWithTx receipts := ev.Receipts @@ -548,7 +557,7 @@ func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*Receip } } else { for i, receipt := range receipts { - if txHashes[receipt.TxHash] { + if _, ok := txHashes[receipt.TxHash]; ok { ret = append(ret, &ReceiptWithTx{ Receipt: receipt, Transaction: txs[i], diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 1d69d57ac8..5788cac53d 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -44,6 +44,7 @@ type Config struct { LogCacheSize int // maximum number of cached blocks (default: 32) Timeout time.Duration // how long filters stay active (default: 5min) LogQueryLimit int // maximum number of addresses allowed in filter criteria (default: 1000) + RangeLimit uint64 // maximum block range allowed in filter criteria (default: 0) } func (cfg Config) withDefaults() Config { @@ -187,9 +188,9 @@ type subscription struct { txs chan []*types.Transaction headers chan *types.Header receipts chan []*ReceiptWithTx - txHashes map[common.Hash]bool // contains transaction hashes for transactionReceipts subscription filtering - installed chan struct{} // closed when the filter is installed - err chan error // closed when the filter is uninstalled + txHashes map[common.Hash]struct{} // contains transaction hashes for transactionReceipts subscription filtering + installed chan struct{} // closed when the filter is installed + err chan error // closed when the filter is uninstalled } // EventSystem creates subscriptions, processes events and broadcasts them to the @@ -405,9 +406,9 @@ func (es *EventSystem) SubscribePendingTxs(txs chan []*types.Transaction) *Subsc // transactions when they are included in blocks. If txHashes is provided, only receipts // for those specific transaction hashes will be delivered. func (es *EventSystem) SubscribeTransactionReceipts(txHashes []common.Hash, receipts chan []*ReceiptWithTx) *Subscription { - hashSet := make(map[common.Hash]bool) + hashSet := make(map[common.Hash]struct{}, len(txHashes)) for _, h := range txHashes { - hashSet[h] = true + hashSet[h] = struct{}{} } sub := &subscription{ id: rpc.NewID(), diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index eeaa1397bf..d503da6ec6 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -550,7 +550,7 @@ func TestExceedLogQueryLimit(t *testing.T) { } ) - _, err := gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index c5a955f13b..41359c1e21 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -109,7 +109,7 @@ func benchmarkFilters(b *testing.B, history uint64, noHistory bool) { backend.startFilterMaps(history, noHistory, filtermaps.DefaultParams) defer backend.stopFilterMaps() - filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil, 0) for b.Loop() { filter.begin = 0 logs, _ := filter.Logs(context.Background()) @@ -205,7 +205,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err = gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } @@ -319,70 +319,70 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}), + f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}, 0), }, { - f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}), + f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}), + f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}), + f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}, 0), }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil, 0), }, { - f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}), + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}, 0), }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), }, { - f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0), err: "safe header not found", }, { - f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, { - f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), err: errPendingLogsUnsupported.Error(), }, } { @@ -405,7 +405,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { } t.Run("timeout", func(t *testing.T) { - f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil) + f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil, 0) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour)) defer cancel() _, err := f.Logs(ctx) @@ -428,7 +428,7 @@ func TestRangeLogs(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } ) - _, err := gspec.Commit(db, triedb.NewDatabase(db, nil)) + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) if err != nil { t.Fatal(err) } @@ -468,7 +468,7 @@ func TestRangeLogs(t *testing.T) { newFilter := func(begin, end int64) { testCase++ event = 0 - filter = sys.NewRangeFilter(begin, end, addresses, nil) + filter = sys.NewRangeFilter(begin, end, addresses, nil, 0) filter.rangeLogsTestHook = make(chan rangeLogsTestEvent) go func(filter *Filter) { filter.Logs(context.Background()) @@ -605,3 +605,41 @@ func TestRangeLogs(t *testing.T) { expEvent(rangeLogsTestReorg, 400, 901) expEvent(rangeLogsTestDone, 0, 0) } + +func TestRangeLimit(t *testing.T) { + db := rawdb.NewMemoryDatabase() + backend, sys := newTestFilterSystem(db, Config{}) + defer db.Close() + + gspec := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + _, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil) + if err != nil { + t.Fatal(err) + } + chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme) + options.TxIndexer = &core.TxIndexerConfig{ + Limit: 0, // index all txs + } + bc, err := core.NewBlockChain(db, nil, gspec, ethash.NewFaker(), options) + if err != nil { + t.Fatal(err) + } + _, err = bc.InsertChain(chain) + if err != nil { + t.Fatal(err) + } + backend.startFilterMaps(0, false, filtermaps.DefaultParams) + defer backend.stopFilterMaps() + + // Set rangeLimit to 5, but request a range of 9 (end - begin = 9, from 0 to 9) + filter := sys.NewRangeFilter(0, 9, nil, nil, 5) + _, err = filter.Logs(context.Background()) + if err == nil || !strings.Contains(err.Error(), "exceed maximum block range") { + t.Fatalf("expected range limit error, got %v", err) + } +} diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index ea30d88ff3..f1b1a67cfc 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/holiman/uint256" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -258,12 +259,12 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit sortedTxs := make([]*types.Transaction, len(txs)) copy(sortedTxs, txs) baseFee := block.BaseFee() + baseFee256 := new(uint256.Int) + if baseFee != nil { + baseFee256.SetFromBig(baseFee) + } slices.SortFunc(sortedTxs, func(a, b *types.Transaction) int { - // It's okay to discard the error because a tx would never be - // accepted into a block with an invalid effective tip. - tip1, _ := a.EffectiveGasTip(baseFee) - tip2, _ := b.EffectiveGasTip(baseFee) - return tip1.Cmp(tip2) + return a.EffectiveGasTipCmp(b, baseFee256) }) var prices []*big.Int diff --git a/eth/handler.go b/eth/handler.go index ff970e2ba6..46634cae88 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -92,6 +92,9 @@ type txPool interface { // can decide whether to receive notifications only for newly seen transactions // or also for reorged out ones. SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription + + // FilterType returns whether the given tx type is supported by the txPool. + FilterType(kind byte) bool } // handlerConfig is the collection of initialization parameters to create a full @@ -111,9 +114,7 @@ type handlerConfig struct { type handler struct { nodeID enode.ID networkID uint64 - - snapSync atomic.Bool // Flag whether snap sync is enabled (gets disabled if we already have blocks) - synced atomic.Bool // Flag whether we're considered synchronised (enables transaction processing) + synced atomic.Bool // Flag whether we're considered synchronised (enables transaction processing) database ethdb.Database txpool txPool @@ -161,40 +162,13 @@ func newHandler(config *handlerConfig) (*handler, error) { handlerDoneCh: make(chan struct{}), handlerStartCh: make(chan struct{}), } - if config.Sync == ethconfig.FullSync { - // The database seems empty as the current block is the genesis. Yet the snap - // block is ahead, so snap sync was enabled for this node at a certain point. - // The scenarios where this can happen is - // * if the user manually (or via a bad block) rolled back a snap sync node - // below the sync point. - // * the last snap sync is not finished while user specifies a full sync this - // time. But we don't have any recent state for full sync. - // In these cases however it's safe to reenable snap sync. - fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock() - if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 { - h.snapSync.Store(true) - log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete") - } else if !h.chain.HasState(fullBlock.Root) { - h.snapSync.Store(true) - log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing") - } - } else { - head := h.chain.CurrentBlock() - if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) { - log.Info("Switch sync mode from snap sync to full sync", "reason", "snap sync complete") - } else { - // If snap sync was requested and our database is empty, grant it - h.snapSync.Store(true) - log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash()) - } - } + // Construct the downloader (long sync) + h.downloader = downloader.New(config.Database, config.Sync, h.eventMux, h.chain, h.removePeer, h.enableSyncedFeatures) + // If snap sync is requested but snapshots are disabled, fail loudly - if h.snapSync.Load() && (config.Chain.Snapshots() == nil && config.Chain.TrieDB().Scheme() == rawdb.HashScheme) { + if h.downloader.ConfigSyncMode() == ethconfig.SnapSync && (config.Chain.Snapshots() == nil && config.Chain.TrieDB().Scheme() == rawdb.HashScheme) { return nil, errors.New("snap sync not supported with snapshots disabled") } - // Construct the downloader (long sync) - h.downloader = downloader.New(config.Database, h.eventMux, h.chain, h.removePeer, h.enableSyncedFeatures) - fetchTx := func(peer string, hashes []common.Hash) error { p := h.peers.peer(peer) if p == nil { @@ -205,7 +179,18 @@ func newHandler(config *handlerConfig) (*handler, error) { addTxs := func(txs []*types.Transaction) []error { return h.txpool.Add(txs, false) } - h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) + + validateMeta := func(tx common.Hash, kind byte) error { + if h.txpool.Has(tx) { + return txpool.ErrAlreadyKnown + } + if !h.txpool.FilterType(kind) { + return types.ErrTxTypeNotSupported + } + return nil + } + + h.txFetcher = fetcher.NewTxFetcher(validateMeta, addTxs, fetchTx, h.removePeer) return h, nil } @@ -267,7 +252,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } reject := false // reserved peer slots - if h.snapSync.Load() { + if h.downloader.ConfigSyncMode() == ethconfig.SnapSync { if snap == nil { // If we are running snap-sync, we want to reserve roughly half the peer // slots for peers supporting the snap protocol. @@ -524,7 +509,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { annCount += len(hashes) peer.AsyncSendPooledTransactionHashes(hashes) } - log.Debug("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs, + log.Trace("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs, "bcastpeers", len(txset), "bcastcount", directCount, "annpeers", len(annos), "anncount", annCount) } @@ -544,15 +529,7 @@ func (h *handler) txBroadcastLoop() { // enableSyncedFeatures enables the post-sync functionalities when the initial // sync is finished. func (h *handler) enableSyncedFeatures() { - // Mark the local node as synced. h.synced.Store(true) - - // If we were running snap sync and it finished, disable doing another - // round on next sync cycle - if h.snapSync.Load() { - log.Info("Snap sync complete, auto disabling") - h.snapSync.Store(false) - } } // blockRangeState holds the state of the block range update broadcasting mechanism. @@ -590,7 +567,7 @@ func (h *handler) blockRangeLoop(st *blockRangeState) { if ev == nil { continue } - if _, ok := ev.Data.(downloader.StartEvent); ok && h.snapSync.Load() { + if _, ok := ev.Data.(downloader.StartEvent); ok && h.downloader.ConfigSyncMode() == ethconfig.SnapSync { h.blockRangeWhileSnapSyncing(st) } case <-st.headCh: diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 11742b14ad..8704a86af4 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -61,19 +62,42 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: - for _, tx := range *packet { - if tx.Type() == types.BlobTxType { - return errors.New("disallowed broadcast blob transaction") - } + txs, err := packet.Items() + if err != nil { + return fmt.Errorf("Transactions: %v", err) + } + if err := handleTransactions(peer, txs, true); err != nil { + return fmt.Errorf("Transactions: %v", err) } - return h.txFetcher.Enqueue(peer.ID(), *packet, false) + return h.txFetcher.Enqueue(peer.ID(), txs, false) - case *eth.PooledTransactionsResponse: - // If we receive any blob transactions missing sidecars, or with - // sidecars that don't correspond to the versioned hashes reported - // in the header, disconnect from the sending peer. - for _, tx := range *packet { - if tx.Type() == types.BlobTxType { + case *eth.PooledTransactionsPacket: + txs, err := packet.List.Items() + if err != nil { + return fmt.Errorf("PooledTransactions: %v", err) + } + if err := handleTransactions(peer, txs, false); err != nil { + return fmt.Errorf("PooledTransactions: %v", err) + } + return h.txFetcher.Enqueue(peer.ID(), txs, true) + + default: + return fmt.Errorf("unexpected eth packet type: %T", packet) + } +} + +// handleTransactions marks all given transactions as known to the peer +// and performs basic validations. +func handleTransactions(peer *eth.Peer, list []*types.Transaction, directBroadcast bool) error { + seen := make(map[common.Hash]struct{}) + for _, tx := range list { + if tx.Type() == types.BlobTxType { + if directBroadcast { + return errors.New("disallowed broadcast blob transaction") + } else { + // If we receive any blob transactions missing sidecars, or with + // sidecars that don't correspond to the versioned hashes reported + // in the header, disconnect from the sending peer. if tx.BlobTxSidecar() == nil { return errors.New("received sidecar-less blob transaction") } @@ -82,9 +106,16 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { } } } - return h.txFetcher.Enqueue(peer.ID(), *packet, true) - default: - return fmt.Errorf("unexpected eth packet type: %T", packet) + // Check for duplicates. + hash := tx.Hash() + if _, exists := seen[hash]; exists { + return fmt.Errorf("multiple copies of the same hash %v", hash) + } + seen[hash] = struct{}{} + + // Mark as known. + peer.MarkTransaction(hash) } + return nil } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 3968ca59d4..3cb58500ba 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -60,11 +60,19 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return nil case *eth.TransactionsPacket: - h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + txs, err := packet.Items() + if err != nil { + return err + } + h.txBroadcasts.Send(txs) return nil - case *eth.PooledTransactionsResponse: - h.txBroadcasts.Send(([]*types.Transaction)(*packet)) + case *eth.PooledTransactionsPacket: + txs, err := packet.List.Items() + if err != nil { + return err + } + h.txBroadcasts.Send(txs) return nil default: @@ -232,7 +240,7 @@ func testRecvTransactions(t *testing.T, protocol uint) { t.Parallel() // Create a message handler, configure it to accept transactions and watch them - handler := newTestHandler() + handler := newTestHandler(ethconfig.FullSync) defer handler.close() handler.handler.synced.Store(true) // mark synced to accept transactions @@ -284,7 +292,7 @@ func testSendTransactions(t *testing.T, protocol uint) { t.Parallel() // Create a message handler and fill the pool with big transactions - handler := newTestHandler() + handler := newTestHandler(ethconfig.FullSync) defer handler.close() insert := make([]*types.Transaction, 100) @@ -365,13 +373,12 @@ func testTransactionPropagation(t *testing.T, protocol uint) { // Create a source handler to send transactions from and a number of sinks // to receive them. We need multiple sinks since a one-to-one peering would // broadcast all transactions without announcement. - source := newTestHandler() - source.handler.snapSync.Store(false) // Avoid requiring snap, otherwise some will be dropped below + source := newTestHandler(ethconfig.FullSync) defer source.close() sinks := make([]*testHandler, 10) for i := 0; i < len(sinks); i++ { - sinks[i] = newTestHandler() + sinks[i] = newTestHandler(ethconfig.FullSync) defer sinks[i].close() sinks[i].handler.synced.Store(true) // mark synced to accept transactions diff --git a/eth/handler_test.go b/eth/handler_test.go index 971873e221..a3e935fd1a 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -163,6 +163,15 @@ func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bo return p.txFeed.Subscribe(ch) } +// FilterType should check whether the pool supports the given type of transactions. +func (p *testTxPool) FilterType(kind byte) bool { + switch kind { + case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: + return true + } + return false +} + // testHandler is a live implementation of the Ethereum protocol handler, just // preinitialized with some sane testing defaults and the transaction pool mocked // out. @@ -174,13 +183,13 @@ type testHandler struct { } // newTestHandler creates a new handler for testing purposes with no blocks. -func newTestHandler() *testHandler { - return newTestHandlerWithBlocks(0) +func newTestHandler(mode ethconfig.SyncMode) *testHandler { + return newTestHandlerWithBlocks(0, mode) } // newTestHandlerWithBlocks creates a new handler for testing purposes, with a // given number of initial blocks. -func newTestHandlerWithBlocks(blocks int) *testHandler { +func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler { // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() gspec := &core.Genesis{ @@ -200,7 +209,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Chain: chain, TxPool: txpool, Network: 1, - Sync: ethconfig.SnapSync, + Sync: mode, BloomCache: 1, }) handler.Start(1000) diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index cba40596fc..3f78fb4646 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" ) var ( @@ -47,9 +48,10 @@ type Request struct { sink chan *Response // Channel to deliver the response on cancel chan struct{} // Channel to cancel requests ahead of time - code uint64 // Message code of the request packet - want uint64 // Message code of the response packet - data interface{} // Data content of the request packet + code uint64 // Message code of the request packet + want uint64 // Message code of the response packet + numItems int // Number of requested items + data interface{} // Data content of the request packet Peer string // Demultiplexer if cross-peer requests are batched together Sent time.Time // Timestamp when the request was sent @@ -190,20 +192,31 @@ func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) erro func (p *Peer) dispatcher() { pending := make(map[uint64]*Request) +loop: for { select { case reqOp := <-p.reqDispatch: req := reqOp.req req.Sent = time.Now() - requestTracker.Track(p.id, p.version, req.code, req.want, req.id) - err := p2p.Send(p.rw, req.code, req.data) - reqOp.fail <- err - - if err == nil { - pending[req.id] = req + treq := tracker.Request{ + ID: req.id, + ReqCode: req.code, + RespCode: req.want, + Size: req.numItems, + } + if err := p.tracker.Track(treq); err != nil { + reqOp.fail <- err + continue loop + } + if err := p2p.Send(p.rw, req.code, req.data); err != nil { + reqOp.fail <- err + continue loop } + pending[req.id] = req + reqOp.fail <- nil + case cancelOp := <-p.reqCancel: // Retrieve the pending request to cancel and short circuit if it // has already been serviced and is not available anymore @@ -220,9 +233,6 @@ func (p *Peer) dispatcher() { res := resOp.res res.Req = pending[res.id] - // Independent if the request exists or not, track this packet - requestTracker.Fulfil(p.id, p.version, res.code, res.id) - switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled @@ -249,6 +259,7 @@ func (p *Peer) dispatcher() { } case <-p.term: + p.tracker.Stop() return } } diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 749b6544ea..f27615fda5 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -23,9 +23,11 @@ import ( "math/big" "math/rand" "os" + "reflect" "testing" "time" + "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -42,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) @@ -360,8 +363,8 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { GetBlockHeadersRequest: tt.query, }) if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, &BlockHeadersPacket{ - RequestId: 123, - BlockHeadersRequest: headers, + RequestId: 123, + List: encodeRL(headers), }); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } @@ -374,7 +377,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { RequestId: 456, GetBlockHeadersRequest: tt.query, }) - expected := &BlockHeadersPacket{RequestId: 456, BlockHeadersRequest: headers} + expected := &BlockHeadersPacket{RequestId: 456, List: encodeRL(headers)} if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, expected); err != nil { t.Errorf("test %d by hash: headers mismatch: %v", i, err) } @@ -437,7 +440,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - bodies []*BlockBody + bodies []BlockBody seen = make(map[int64]bool) ) for j := 0; j < tt.random; j++ { @@ -449,7 +452,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { block := backend.chain.GetBlockByNumber(uint64(num)) hashes = append(hashes, block.Hash()) if len(bodies) < tt.expected { - bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles(), Withdrawals: block.Withdrawals()}) + bodies = append(bodies, encodeBody(block)) } break } @@ -459,7 +462,7 @@ func testGetBlockBodies(t *testing.T, protocol uint) { hashes = append(hashes, hash) if tt.available[j] && len(bodies) < tt.expected { block := backend.chain.GetBlockByHash(hash) - bodies = append(bodies, &BlockBody{Transactions: block.Transactions(), Uncles: block.Uncles(), Withdrawals: block.Withdrawals()}) + bodies = append(bodies, encodeBody(block)) } } @@ -469,14 +472,69 @@ func testGetBlockBodies(t *testing.T, protocol uint) { GetBlockBodiesRequest: hashes, }) if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, &BlockBodiesPacket{ - RequestId: 123, - BlockBodiesResponse: bodies, + RequestId: 123, + List: encodeRL(bodies), }); err != nil { t.Fatalf("test %d: bodies mismatch: %v", i, err) } } } +func encodeBody(b *types.Block) BlockBody { + body := BlockBody{ + Transactions: encodeRL([]*types.Transaction(b.Transactions())), + Uncles: encodeRL(b.Uncles()), + } + if b.Withdrawals() != nil { + wd := encodeRL([]*types.Withdrawal(b.Withdrawals())) + body.Withdrawals = &wd + } + return body +} + +func TestHashBody(t *testing.T) { + key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + signer := types.NewCancunSigner(big.NewInt(1)) + + // create block 1 + header := &types.Header{Number: big.NewInt(11)} + txs := []*types.Transaction{ + types.MustSignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: big.NewInt(1), + Nonce: 1, + Data: []byte("testing"), + }), + types.MustSignNewTx(key, signer, &types.LegacyTx{ + Nonce: 2, + Data: []byte("testing"), + }), + } + uncles := []*types.Header{{Number: big.NewInt(10)}} + body1 := &types.Body{Transactions: txs, Uncles: uncles} + block1 := types.NewBlock(header, body1, nil, trie.NewStackTrie(nil)) + + // create block 2 (has withdrawals) + header2 := &types.Header{Number: big.NewInt(12)} + body2 := &types.Body{ + Withdrawals: []*types.Withdrawal{{Index: 10}, {Index: 11}}, + } + block2 := types.NewBlock(header2, body2, nil, trie.NewStackTrie(nil)) + + expectedHashes := BlockBodyHashes{ + TransactionRoots: []common.Hash{block1.TxHash(), block2.TxHash()}, + WithdrawalRoots: []common.Hash{common.Hash{}, *block2.Header().WithdrawalsHash}, + UncleHashes: []common.Hash{block1.UncleHash(), block2.UncleHash()}, + } + + // compute hash like protocol handler does + protocolBodies := []BlockBody{encodeBody(block1), encodeBody(block2)} + hashes := hashBodyParts(protocolBodies) + if !reflect.DeepEqual(hashes, expectedHashes) { + t.Errorf("wrong hashes: %s", spew.Sdump(hashes)) + t.Logf("expected: %s", spew.Sdump(expectedHashes)) + } +} + // Tests that the transaction receipts can be retrieved based on hashes. func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } @@ -528,13 +586,13 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - receipts []*ReceiptList68 + receipts rlp.RawList[*ReceiptList68] ) for i := uint64(0); i <= backend.chain.CurrentBlock().Number.Uint64(); i++ { block := backend.chain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) trs := backend.chain.GetReceiptsByHash(block.Hash()) - receipts = append(receipts, NewReceiptList68(trs)) + receipts.Append(NewReceiptList68(trs)) } // Send the hash request and verify the response @@ -688,10 +746,18 @@ func testGetPooledTransaction(t *testing.T, blobTx bool) { RequestId: 123, GetPooledTransactionsRequest: []common.Hash{tx.Hash()}, }) - if err := p2p.ExpectMsg(peer.app, PooledTransactionsMsg, PooledTransactionsPacket{ - RequestId: 123, - PooledTransactionsResponse: []*types.Transaction{tx}, + if err := p2p.ExpectMsg(peer.app, PooledTransactionsMsg, &PooledTransactionsPacket{ + RequestId: 123, + List: encodeRL([]*types.Transaction{tx}), }); err != nil { t.Errorf("pooled transaction mismatch: %v", err) } } + +func encodeRL[T any](slice []T) rlp.RawList[T] { + rl, err := rlp.EncodeToRawList(slice) + if err != nil { + panic(err) + } + return rl +} diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index aad3353d88..7f1ccc360d 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -17,23 +17,22 @@ package eth import ( + "bytes" "encoding/json" "errors" "fmt" - "time" + "math" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) -// requestTracker is a singleton tracker for eth/66 and newer request times. -var requestTracker = tracker.New(ProtocolName, 5*time.Minute) - func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query var query GetBlockHeadersPacket @@ -356,9 +355,18 @@ func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return err } + tresp := tracker.Response{ID: res.RequestId, MsgCode: BlockHeadersMsg, Size: res.List.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("BlockHeaders: %w", err) + } + headers, err := res.List.Items() + if err != nil { + return fmt.Errorf("BlockHeaders: %w", err) + } + metadata := func() interface{} { - hashes := make([]common.Hash, len(res.BlockHeadersRequest)) - for i, header := range res.BlockHeadersRequest { + hashes := make([]common.Hash, len(headers)) + for i, header := range headers { hashes[i] = header.Hash() } return hashes @@ -366,7 +374,7 @@ func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { return peer.dispatchResponse(&Response{ id: res.RequestId, code: BlockHeadersMsg, - Res: &res.BlockHeadersRequest, + Res: (*BlockHeadersRequest)(&headers), }, metadata) } @@ -376,53 +384,150 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(res); err != nil { return err } - metadata := func() interface{} { - var ( - txsHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - uncleHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - withdrawalHashes = make([]common.Hash, len(res.BlockBodiesResponse)) - ) - hasher := trie.NewStackTrie(nil) - for i, body := range res.BlockBodiesResponse { - txsHashes[i] = types.DeriveSha(types.Transactions(body.Transactions), hasher) - uncleHashes[i] = types.CalcUncleHash(body.Uncles) - if body.Withdrawals != nil { - withdrawalHashes[i] = types.DeriveSha(types.Withdrawals(body.Withdrawals), hasher) - } - } - return [][]common.Hash{txsHashes, uncleHashes, withdrawalHashes} + + // Check against the request. + length := res.List.Len() + tresp := tracker.Response{ID: res.RequestId, MsgCode: BlockBodiesMsg, Size: length} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("BlockBodies: %w", err) + } + + // Collect items and dispatch. + items, err := res.List.Items() + if err != nil { + return fmt.Errorf("BlockBodies: %w", err) } + metadata := func() any { return hashBodyParts(items) } return peer.dispatchResponse(&Response{ id: res.RequestId, code: BlockBodiesMsg, - Res: &res.BlockBodiesResponse, + Res: (*BlockBodiesResponse)(&items), }, metadata) } +// BlockBodyHashes contains the lists of block body part roots for a list of block bodies. +type BlockBodyHashes struct { + TransactionRoots []common.Hash + WithdrawalRoots []common.Hash + UncleHashes []common.Hash +} + +func hashBodyParts(items []BlockBody) BlockBodyHashes { + h := BlockBodyHashes{ + TransactionRoots: make([]common.Hash, len(items)), + WithdrawalRoots: make([]common.Hash, len(items)), + UncleHashes: make([]common.Hash, len(items)), + } + hasher := trie.NewStackTrie(nil) + for i, body := range items { + // txs + txsList := newDerivableRawList(&body.Transactions, writeTxForHash) + h.TransactionRoots[i] = types.DeriveSha(txsList, hasher) + // uncles + if body.Uncles.Len() == 0 { + h.UncleHashes[i] = types.EmptyUncleHash + } else { + h.UncleHashes[i] = crypto.Keccak256Hash(body.Uncles.Bytes()) + } + // withdrawals + if body.Withdrawals != nil { + wdlist := newDerivableRawList(body.Withdrawals, nil) + h.WithdrawalRoots[i] = types.DeriveSha(wdlist, hasher) + } + } + return h +} + +// derivableRawList implements types.DerivableList for a serialized RLP list. +type derivableRawList struct { + data []byte + offsets []uint32 + write func([]byte, *bytes.Buffer) +} + +func newDerivableRawList[T any](list *rlp.RawList[T], write func([]byte, *bytes.Buffer)) *derivableRawList { + dl := derivableRawList{data: list.Content(), write: write} + if dl.write == nil { + // default transform is identity + dl.write = func(b []byte, buf *bytes.Buffer) { buf.Write(b) } + } + // Assert to ensure 32-bit offsets are valid. This can never trigger + // unless a block body component or p2p receipt list is larger than 4GB. + if uint(len(dl.data)) > math.MaxUint32 { + panic("list data too big for derivableRawList") + } + it := list.ContentIterator() + dl.offsets = make([]uint32, list.Len()) + for i := 0; it.Next(); i++ { + dl.offsets[i] = uint32(it.Offset()) + } + return &dl +} + +// Len returns the number of items in the list. +func (dl *derivableRawList) Len() int { + return len(dl.offsets) +} + +// EncodeIndex writes the i'th item to the buffer. +func (dl *derivableRawList) EncodeIndex(i int, buf *bytes.Buffer) { + start := dl.offsets[i] + end := uint32(len(dl.data)) + if i != len(dl.offsets)-1 { + end = dl.offsets[i+1] + } + dl.write(dl.data[start:end], buf) +} + +// writeTxForHash changes a transaction in 'network encoding' into the format used for +// the transactions MPT. +func writeTxForHash(tx []byte, buf *bytes.Buffer) { + k, content, _, _ := rlp.Split(tx) + if k == rlp.List { + buf.Write(tx) // legacy tx + } else { + buf.Write(content) // typed tx + } +} + func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket[L]) if err := msg.Decode(res); err != nil { return err } + + tresp := tracker.Response{ID: res.RequestId, MsgCode: ReceiptsMsg, Size: res.List.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("Receipts: %w", err) + } + // Assign temporary hashing buffer to each list item, the same buffer is shared // between all receipt list instances. + receiptLists, err := res.List.Items() + if err != nil { + return fmt.Errorf("Receipts: %w", err) + } buffers := new(receiptListBuffers) - for i := range res.List { - res.List[i].setBuffers(buffers) + for i := range receiptLists { + receiptLists[i].setBuffers(buffers) } metadata := func() interface{} { hasher := trie.NewStackTrie(nil) - hashes := make([]common.Hash, len(res.List)) - for i := range res.List { - hashes[i] = types.DeriveSha(res.List[i], hasher) + hashes := make([]common.Hash, len(receiptLists)) + for i := range receiptLists { + hashes[i] = types.DeriveSha(receiptLists[i].Derivable(), hasher) } return hashes } var enc ReceiptsRLPResponse - for i := range res.List { - enc = append(enc, res.List[i].EncodeForStorage()) + for i := range receiptLists { + encReceipts, err := receiptLists[i].EncodeForStorage() + if err != nil { + return fmt.Errorf("Receipts: invalid list %d: %v", i, err) + } + enc = append(enc, encReceipts) } return peer.dispatchResponse(&Response{ id: res.RequestId, @@ -446,7 +551,7 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) } // Schedule all the unknown hashes for retrieval for _, hash := range ann.Hashes { - peer.markTransaction(hash) + peer.MarkTransaction(hash) } return backend.Handle(peer, ann) } @@ -494,19 +599,8 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { if err := msg.Decode(&txs); err != nil { return err } - // Duplicate transactions are not allowed - seen := make(map[common.Hash]struct{}) - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("Transactions: transaction %d is nil", i) - } - hash := tx.Hash() - if _, exists := seen[hash]; exists { - return fmt.Errorf("Transactions: multiple copies of the same hash %v", hash) - } - seen[hash] = struct{}{} - peer.markTransaction(hash) + if txs.Len() > maxTransactionAnnouncements { + return fmt.Errorf("too many transactions") } return backend.Handle(peer, &txs) } @@ -516,28 +610,22 @@ func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { if !backend.AcceptTxs() { return nil } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { + + // Check against request and decode. + var resp PooledTransactionsPacket + if err := msg.Decode(&resp); err != nil { return err } - // Duplicate transactions are not allowed - seen := make(map[common.Hash]struct{}) - for i, tx := range txs.PooledTransactionsResponse { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("PooledTransactions: transaction %d is nil", i) - } - hash := tx.Hash() - if _, exists := seen[hash]; exists { - return fmt.Errorf("PooledTransactions: multiple copies of the same hash %v", hash) - } - seen[hash] = struct{}{} - peer.markTransaction(hash) + tresp := tracker.Response{ + ID: resp.RequestId, + MsgCode: PooledTransactionsMsg, + Size: resp.List.Len(), + } + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("PooledTransactions: %w", err) } - requestTracker.Fulfil(peer.id, peer.version, PooledTransactionsMsg, txs.RequestId) - return backend.Handle(peer, &txs.PooledTransactionsResponse) + return backend.Handle(peer, &resp) } func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 40c54a3570..4ea2d7158c 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -19,11 +19,13 @@ package eth import ( "math/rand" "sync/atomic" + "time" mapset "github.com/deckarep/golang-set/v2" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,9 +45,10 @@ const ( // Peer is a collection of relevant information we have about a `eth` peer. type Peer struct { + *p2p.Peer // The embedded P2P package peer + id string // Unique ID for the peer, cached - *p2p.Peer // The embedded P2P package peer rw p2p.MsgReadWriter // Input/output streams for snap version uint // Protocol version negotiated lastRange atomic.Pointer[BlockRangeUpdatePacket] @@ -55,6 +58,7 @@ type Peer struct { txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + tracker *tracker.Tracker reqDispatch chan *request // Dispatch channel to send requests and track then until fulfillment reqCancel chan *cancel // Dispatch channel to cancel pending requests and untrack them resDispatch chan *response // Dispatch channel to fulfil pending requests and untrack them @@ -65,14 +69,17 @@ type Peer struct { // NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} + id := p.ID().String() peer := &Peer{ - id: p.ID().String(), + id: id, Peer: p, rw: rw, version: version, knownTxs: newKnownCache(maxKnownTxs), txBroadcast: make(chan []common.Hash), txAnnounce: make(chan []common.Hash), + tracker: tracker.New(cap, id, 5*time.Minute), reqDispatch: make(chan *request), reqCancel: make(chan *cancel), resDispatch: make(chan *response), @@ -115,9 +122,9 @@ func (p *Peer) KnownTransaction(hash common.Hash) bool { return p.knownTxs.Contains(hash) } -// markTransaction marks a transaction as known for the peer, ensuring that it +// MarkTransaction marks a transaction as known for the peer, ensuring that it // will never be propagated to this particular peer. -func (p *Peer) markTransaction(hash common.Hash) { +func (p *Peer) MarkTransaction(hash common.Hash) { // If we reached the memory allowance, drop a previously known transaction hash p.knownTxs.Add(hash) } @@ -222,10 +229,11 @@ func (p *Peer) RequestOneHeader(hash common.Hash, sink chan *Response) (*Request id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: 1, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -249,10 +257,11 @@ func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, re id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: amount, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -276,10 +285,11 @@ func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, rever id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockHeadersMsg, - want: BlockHeadersMsg, + id: id, + sink: sink, + code: GetBlockHeadersMsg, + want: BlockHeadersMsg, + numItems: amount, data: &GetBlockHeadersPacket{ RequestId: id, GetBlockHeadersRequest: &GetBlockHeadersRequest{ @@ -303,10 +313,11 @@ func (p *Peer) RequestBodies(hashes []common.Hash, sink chan *Response) (*Reques id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetBlockBodiesMsg, - want: BlockBodiesMsg, + id: id, + sink: sink, + code: GetBlockBodiesMsg, + want: BlockBodiesMsg, + numItems: len(hashes), data: &GetBlockBodiesPacket{ RequestId: id, GetBlockBodiesRequest: hashes, @@ -324,10 +335,11 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ id := rand.Uint64() req := &Request{ - id: id, - sink: sink, - code: GetReceiptsMsg, - want: ReceiptsMsg, + id: id, + sink: sink, + code: GetReceiptsMsg, + want: ReceiptsMsg, + numItems: len(hashes), data: &GetReceiptsPacket{ RequestId: id, GetReceiptsRequest: hashes, @@ -341,10 +353,18 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + p.Log().Trace("Fetching batch of transactions", "count", len(hashes)) id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + err := p.tracker.Track(tracker.Request{ + ID: id, + ReqCode: GetPooledTransactionsMsg, + RespCode: PooledTransactionsMsg, + Size: len(hashes), + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket{ RequestId: id, GetPooledTransactionsRequest: hashes, diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 7c41e7a996..6ab800f4f4 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -49,6 +49,9 @@ var protocolLengths = map[uint]uint64{ETH68: 17, ETH69: 18} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 +// This is the maximum number of transactions in a Transactions message. +const maxTransactionAnnouncements = 5000 + const ( StatusMsg = 0x00 NewBlockHashesMsg = 0x01 @@ -127,7 +130,9 @@ func (p *NewBlockHashesPacket) Unpack() ([]common.Hash, []uint64) { } // TransactionsPacket is the network packet for broadcasting new transactions. -type TransactionsPacket []*types.Transaction +type TransactionsPacket struct { + rlp.RawList[*types.Transaction] +} // GetBlockHeadersRequest represents a block header query. type GetBlockHeadersRequest struct { @@ -185,7 +190,7 @@ type BlockHeadersRequest []*types.Header // BlockHeadersPacket represents a block header response over with request ID wrapping. type BlockHeadersPacket struct { RequestId uint64 - BlockHeadersRequest + List rlp.RawList[*types.Header] } // BlockHeadersRLPResponse represents a block header response, to use when we already @@ -213,14 +218,11 @@ type GetBlockBodiesPacket struct { GetBlockBodiesRequest } -// BlockBodiesResponse is the network packet for block content distribution. -type BlockBodiesResponse []*BlockBody - // BlockBodiesPacket is the network packet for block content distribution with // request ID wrapping. type BlockBodiesPacket struct { RequestId uint64 - BlockBodiesResponse + List rlp.RawList[BlockBody] } // BlockBodiesRLPResponse is used for replying to block body requests, in cases @@ -234,25 +236,14 @@ type BlockBodiesRLPPacket struct { BlockBodiesRLPResponse } +// BlockBodiesResponse is the network packet for block content distribution. +type BlockBodiesResponse []BlockBody + // BlockBody represents the data content of a single block. type BlockBody struct { - Transactions []*types.Transaction // Transactions contained within a block - Uncles []*types.Header // Uncles contained within a block - Withdrawals []*types.Withdrawal `rlp:"optional"` // Withdrawals contained within a block -} - -// Unpack retrieves the transactions and uncles from the range packet and returns -// them in a split flat format that's more consistent with the internal data structures. -func (p *BlockBodiesResponse) Unpack() ([][]*types.Transaction, [][]*types.Header, [][]*types.Withdrawal) { - var ( - txset = make([][]*types.Transaction, len(*p)) - uncleset = make([][]*types.Header, len(*p)) - withdrawalset = make([][]*types.Withdrawal, len(*p)) - ) - for i, body := range *p { - txset[i], uncleset[i], withdrawalset[i] = body.Transactions, body.Uncles, body.Withdrawals - } - return txset, uncleset, withdrawalset + Transactions rlp.RawList[*types.Transaction] + Uncles rlp.RawList[*types.Header] + Withdrawals *rlp.RawList[*types.Withdrawal] `rlp:"optional"` } // GetReceiptsRequest represents a block receipts query. @@ -271,15 +262,15 @@ type ReceiptsResponse []types.Receipts type ReceiptsList interface { *ReceiptList68 | *ReceiptList69 setBuffers(*receiptListBuffers) - EncodeForStorage() rlp.RawValue - types.DerivableList + EncodeForStorage() (rlp.RawValue, error) + Derivable() types.DerivableList } // ReceiptsPacket is the network packet for block receipts distribution with // request ID wrapping. type ReceiptsPacket[L ReceiptsList] struct { RequestId uint64 - List []L + List rlp.RawList[L] } // ReceiptsRLPResponse is used for receipts, when we already have it encoded @@ -314,7 +305,7 @@ type PooledTransactionsResponse []*types.Transaction // with request ID wrapping. type PooledTransactionsPacket struct { RequestId uint64 - PooledTransactionsResponse + List rlp.RawList[*types.Transaction] } // PooledTransactionsRLPResponse is the network packet for transaction distribution, used @@ -367,8 +358,8 @@ func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransac func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } -func (*PooledTransactionsResponse) Name() string { return "PooledTransactions" } -func (*PooledTransactionsResponse) Kind() byte { return PooledTransactionsMsg } +func (*PooledTransactionsPacket) Name() string { return "PooledTransactions" } +func (*PooledTransactionsPacket) Kind() byte { return PooledTransactionsMsg } func (*GetReceiptsRequest) Name() string { return "GetReceipts" } func (*GetReceiptsRequest) Kind() byte { return GetReceiptsMsg } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index b402cba549..28231e4f47 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -78,34 +78,34 @@ func TestEmptyMessages(t *testing.T) { for i, msg := range []any{ // Headers GetBlockHeadersPacket{1111, nil}, - BlockHeadersPacket{1111, nil}, // Bodies GetBlockBodiesPacket{1111, nil}, - BlockBodiesPacket{1111, nil}, BlockBodiesRLPPacket{1111, nil}, // Receipts GetReceiptsPacket{1111, nil}, // Transactions GetPooledTransactionsPacket{1111, nil}, - PooledTransactionsPacket{1111, nil}, PooledTransactionsRLPPacket{1111, nil}, // Headers - BlockHeadersPacket{1111, BlockHeadersRequest([]*types.Header{})}, + BlockHeadersPacket{1111, encodeRL([]*types.Header{})}, // Bodies GetBlockBodiesPacket{1111, GetBlockBodiesRequest([]common.Hash{})}, - BlockBodiesPacket{1111, BlockBodiesResponse([]*BlockBody{})}, + BlockBodiesPacket{1111, encodeRL([]BlockBody{})}, BlockBodiesRLPPacket{1111, BlockBodiesRLPResponse([]rlp.RawValue{})}, // Receipts GetReceiptsPacket{1111, GetReceiptsRequest([]common.Hash{})}, - ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{}}, - ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{}}, + ReceiptsPacket[*ReceiptList68]{1111, encodeRL([]*ReceiptList68{})}, + ReceiptsPacket[*ReceiptList69]{1111, encodeRL([]*ReceiptList69{})}, // Transactions GetPooledTransactionsPacket{1111, GetPooledTransactionsRequest([]common.Hash{})}, - PooledTransactionsPacket{1111, PooledTransactionsResponse([]*types.Transaction{})}, + PooledTransactionsPacket{1111, encodeRL([]*types.Transaction{})}, PooledTransactionsRLPPacket{1111, PooledTransactionsRLPResponse([]rlp.RawValue{})}, } { - if have, _ := rlp.EncodeToBytes(msg); !bytes.Equal(have, want) { + have, err := rlp.EncodeToBytes(msg) + if err != nil { + t.Errorf("test %d, type %T, error: %v", i, msg, err) + } else if !bytes.Equal(have, want) { t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) } } @@ -116,7 +116,7 @@ func TestMessages(t *testing.T) { // Some basic structs used during testing var ( header *types.Header - blockBody *BlockBody + blockBody BlockBody blockBodyRlp rlp.RawValue txs []*types.Transaction txRlps []rlp.RawValue @@ -150,9 +150,9 @@ func TestMessages(t *testing.T) { } } // init the block body data, both object and rlp form - blockBody = &BlockBody{ - Transactions: txs, - Uncles: []*types.Header{header}, + blockBody = BlockBody{ + Transactions: encodeRL(txs), + Uncles: encodeRL([]*types.Header{header}), } blockBodyRlp, err = rlp.EncodeToBytes(blockBody) if err != nil { @@ -211,7 +211,7 @@ func TestMessages(t *testing.T) { common.FromHex("ca820457c682270f050580"), }, { - BlockHeadersPacket{1111, BlockHeadersRequest{header}}, + BlockHeadersPacket{1111, encodeRL([]*types.Header{header})}, common.FromHex("f90202820457f901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), }, { @@ -219,7 +219,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - BlockBodiesPacket{1111, BlockBodiesResponse([]*BlockBody{blockBody})}, + BlockBodiesPacket{1111, encodeRL([]BlockBody{blockBody})}, common.FromHex("f902dc820457f902d6f902d3f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afbf901fcf901f9a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008208ae820d0582115c8215b3821a0a827788a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"), }, { // Identical to non-rlp-shortcut version @@ -231,7 +231,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{NewReceiptList68(receipts)}}, + ReceiptsPacket[*ReceiptList68]{1111, encodeRL([]*ReceiptList68{NewReceiptList68(receipts)})}, common.FromHex("f902e8820457f902e2f902dff901698082014d80b9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9017001f9016c018201bc80b9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), }, { @@ -240,7 +240,7 @@ func TestMessages(t *testing.T) { common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), }, { - ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{NewReceiptList69(receipts)}}, + ReceiptsPacket[*ReceiptList69]{1111, encodeRL([]*ReceiptList69{NewReceiptList69(receipts)})}, common.FromHex("f8dc820457f8d7f8d5f867808082014d80f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100fff86a01018201bc80f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), }, { @@ -248,7 +248,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - PooledTransactionsPacket{1111, PooledTransactionsResponse(txs)}, + PooledTransactionsPacket{1111, encodeRL(txs)}, common.FromHex("f8d7820457f8d2f867088504a817c8088302e2489435353535353535353535353535353535353535358202008025a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c12a064b1702d9298fee62dfeccc57d322a463ad55ca201256d01f62b45b2e1c21c10f867098504a817c809830334509435353535353535353535353535353535353535358202d98025a052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afba052f8f61201b2b11a78d6e866abc9c3db2ae8631fa656bfe5cb53668255367afb"), }, { diff --git a/eth/protocols/eth/receipt.go b/eth/protocols/eth/receipt.go index b59d7ff4e1..42c7cb75d9 100644 --- a/eth/protocols/eth/receipt.go +++ b/eth/protocols/eth/receipt.go @@ -51,8 +51,8 @@ func newReceipt(tr *types.Receipt) Receipt { } // decode68 parses a receipt in the eth/68 network encoding. -func (r *Receipt) decode68(buf *receiptListBuffers, s *rlp.Stream) error { - k, size, err := s.Kind() +func (r *Receipt) decode68(b []byte) error { + k, content, _, err := rlp.Split(b) if err != nil { return err } @@ -60,69 +60,88 @@ func (r *Receipt) decode68(buf *receiptListBuffers, s *rlp.Stream) error { *r = Receipt{} if k == rlp.List { // Legacy receipt. - return r.decodeInnerList(s, false, true) + return r.decodeInnerList(b, false, true) } // Typed receipt. - if size < 2 || size > maxReceiptSize { - return fmt.Errorf("invalid receipt size %d", size) + if len(content) < 2 || len(content) > maxReceiptSize { + return fmt.Errorf("invalid receipt size %d", len(content)) } - buf.tmp.Reset() - buf.tmp.Grow(int(size)) - payload := buf.tmp.Bytes()[:int(size)] - if err := s.ReadBytes(payload); err != nil { - return err - } - r.TxType = payload[0] - s2 := rlp.NewStream(bytes.NewReader(payload[1:]), 0) - return r.decodeInnerList(s2, false, true) + r.TxType = content[0] + return r.decodeInnerList(content[1:], false, true) } // decode69 parses a receipt in the eth/69 network encoding. -func (r *Receipt) decode69(s *rlp.Stream) error { +func (r *Receipt) decode69(b []byte) error { *r = Receipt{} - return r.decodeInnerList(s, true, false) + return r.decodeInnerList(b, true, false) } // decodeDatabase parses a receipt in the basic database encoding. -func (r *Receipt) decodeDatabase(txType byte, s *rlp.Stream) error { +func (r *Receipt) decodeDatabase(txType byte, b []byte) error { *r = Receipt{TxType: txType} - return r.decodeInnerList(s, false, false) + return r.decodeInnerList(b, false, false) } -func (r *Receipt) decodeInnerList(s *rlp.Stream, readTxType, readBloom bool) error { - _, err := s.List() +func (r *Receipt) decodeInnerList(input []byte, readTxType, readBloom bool) error { + input, _, err := rlp.SplitList(input) if err != nil { - return err + return fmt.Errorf("inner list: %v", err) } + + // txType if readTxType { - r.TxType, err = s.Uint8() + var txType uint64 + txType, input, err = rlp.SplitUint64(input) if err != nil { return fmt.Errorf("invalid txType: %w", err) } + if txType > 0x7f { + return fmt.Errorf("invalid txType: too large") + } + r.TxType = byte(txType) } - r.PostStateOrStatus, err = s.Bytes() + + // status + r.PostStateOrStatus, input, err = rlp.SplitString(input) if err != nil { return fmt.Errorf("invalid postStateOrStatus: %w", err) } - r.GasUsed, err = s.Uint64() + if len(r.PostStateOrStatus) > 1 && len(r.PostStateOrStatus) != 32 { + return fmt.Errorf("invalid postStateOrStatus length %d", len(r.PostStateOrStatus)) + } + + // gas + r.GasUsed, input, err = rlp.SplitUint64(input) if err != nil { return fmt.Errorf("invalid gasUsed: %w", err) } - r.GasUsedForL1, err = s.Uint64() + r.GasUsedForL1, input, err = rlp.SplitUint64(input) if err != nil { return fmt.Errorf("invalid gasUsedForL1: %w", err) } + + // bloom if readBloom { - var b types.Bloom - if err := s.ReadBytes(b[:]); err != nil { + var bloomBytes []byte + bloomBytes, input, err = rlp.SplitString(input) + if err != nil { return fmt.Errorf("invalid bloom: %v", err) } + if len(bloomBytes) != types.BloomByteLength { + return fmt.Errorf("invalid bloom length %d", len(bloomBytes)) + } } - r.Logs, err = s.Raw() + + // logs + _, rest, err := rlp.SplitList(input) if err != nil { return fmt.Errorf("invalid logs: %w", err) } - return s.ListEnd() + if len(rest) != 0 { + return fmt.Errorf("junk at end of receipt") + } + r.Logs = input + return nil } // encodeForStorage produces the the storage encoding, i.e. the result matches @@ -231,32 +250,45 @@ func initBuffers(buf **receiptListBuffers) { } // encodeForStorage encodes a list of receipts for the database. -func (buf *receiptListBuffers) encodeForStorage(rs []Receipt) rlp.RawValue { +func (buf *receiptListBuffers) encodeForStorage(rs rlp.RawList[Receipt], decode func([]byte, *Receipt) error) (rlp.RawValue, error) { var out bytes.Buffer w := &buf.enc w.Reset(&out) outer := w.List() - for _, receipts := range rs { - receipts.encodeForStorage(w) + it := rs.ContentIterator() + for it.Next() { + var receipt Receipt + if err := decode(it.Value(), &receipt); err != nil { + return nil, err + } + receipt.encodeForStorage(w) + } + if it.Err() != nil { + return nil, fmt.Errorf("bad list: %v", it.Err()) } w.ListEnd(outer) w.Flush() - return out.Bytes() + return out.Bytes(), nil } // ReceiptList68 is a block receipt list as downloaded by eth/68. // This also implements types.DerivableList for validation purposes. type ReceiptList68 struct { buf *receiptListBuffers - items []Receipt + items rlp.RawList[Receipt] } // NewReceiptList68 creates a receipt list. // This is slow, and exists for testing purposes. func NewReceiptList68(trs []*types.Receipt) *ReceiptList68 { - rl := &ReceiptList68{items: make([]Receipt, len(trs))} - for i, tr := range trs { - rl.items[i] = newReceipt(tr) + rl := new(ReceiptList68) + initBuffers(&rl.buf) + enc := rlp.NewEncoderBuffer(nil) + for _, tr := range trs { + r := newReceipt(tr) + r.encodeForNetwork68(rl.buf, &enc) + rl.items.AppendRaw(enc.ToBytes()) + enc.Reset(nil) } return rl } @@ -274,17 +306,12 @@ func blockReceiptsToNetwork68(blockReceipts, blockBody rlp.RawValue) ([]byte, er buf receiptListBuffers ) blockReceiptIter, _ := rlp.NewListIterator(blockReceipts) - innerReader := bytes.NewReader(nil) - innerStream := rlp.NewStream(innerReader, 0) w := rlp.NewEncoderBuffer(&out) outer := w.List() for i := 0; blockReceiptIter.Next(); i++ { - content := blockReceiptIter.Value() - innerReader.Reset(content) - innerStream.Reset(innerReader, uint64(len(content))) - var r Receipt txType, _ := nextTxType() - if err := r.decodeDatabase(txType, innerStream); err != nil { + var r Receipt + if err := r.decodeDatabase(txType, blockReceiptIter.Value()); err != nil { return nil, fmt.Errorf("invalid database receipt %d: %v", i, err) } r.encodeForNetwork68(&buf, &w) @@ -300,64 +327,51 @@ func (rl *ReceiptList68) setBuffers(buf *receiptListBuffers) { } // EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList68) EncodeForStorage() rlp.RawValue { +func (rl *ReceiptList68) EncodeForStorage() (rlp.RawValue, error) { initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items) -} - -// Len implements types.DerivableList. -func (rl *ReceiptList68) Len() int { - return len(rl.items) + return rl.buf.encodeForStorage(rl.items, func(data []byte, r *Receipt) error { + return r.decode68(data) + }) } -// EncodeIndex implements types.DerivableList. -func (rl *ReceiptList68) EncodeIndex(i int, out *bytes.Buffer) { +// Derivable turns the receipts into a list that can derive the root hash. +func (rl *ReceiptList68) Derivable() types.DerivableList { initBuffers(&rl.buf) - rl.items[i].encodeForHash(rl.buf, out) + return newDerivableRawList(&rl.items, func(data []byte, outbuf *bytes.Buffer) { + var r Receipt + if r.decode68(data) == nil { + r.encodeForHash(rl.buf, outbuf) + } + }) } // DecodeRLP decodes a list of receipts from the network format. func (rl *ReceiptList68) DecodeRLP(s *rlp.Stream) error { - initBuffers(&rl.buf) - if _, err := s.List(); err != nil { - return err - } - for i := 0; s.MoreDataInList(); i++ { - var item Receipt - err := item.decode68(rl.buf, s) - if err != nil { - return fmt.Errorf("receipt %d: %v", i, err) - } - rl.items = append(rl.items, item) - } - return s.ListEnd() + return rl.items.DecodeRLP(s) } // EncodeRLP encodes the list into the network format of eth/68. -func (rl *ReceiptList68) EncodeRLP(_w io.Writer) error { - initBuffers(&rl.buf) - w := rlp.NewEncoderBuffer(_w) - outer := w.List() - for i := range rl.items { - rl.items[i].encodeForNetwork68(rl.buf, &w) - } - w.ListEnd(outer) - return w.Flush() +func (rl *ReceiptList68) EncodeRLP(w io.Writer) error { + return rl.items.EncodeRLP(w) } // ReceiptList69 is the block receipt list as downloaded by eth/69. // This implements types.DerivableList for validation purposes. type ReceiptList69 struct { buf *receiptListBuffers - items []Receipt + items rlp.RawList[Receipt] } // NewReceiptList69 creates a receipt list. // This is slow, and exists for testing purposes. func NewReceiptList69(trs []*types.Receipt) *ReceiptList69 { - rl := &ReceiptList69{items: make([]Receipt, len(trs))} - for i, tr := range trs { - rl.items[i] = newReceipt(tr) + rl := new(ReceiptList69) + enc := rlp.NewEncoderBuffer(nil) + for _, tr := range trs { + r := newReceipt(tr) + r.encodeForNetwork69(&enc) + rl.items.AppendRaw(enc.ToBytes()) + enc.Reset(nil) } return rl } @@ -368,47 +382,32 @@ func (rl *ReceiptList69) setBuffers(buf *receiptListBuffers) { } // EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList69) EncodeForStorage() rlp.RawValue { +func (rl *ReceiptList69) EncodeForStorage() (rlp.RawValue, error) { initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items) -} - -// Len implements types.DerivableList. -func (rl *ReceiptList69) Len() int { - return len(rl.items) + return rl.buf.encodeForStorage(rl.items, func(data []byte, r *Receipt) error { + return r.decode69(data) + }) } -// EncodeIndex implements types.DerivableList. -func (rl *ReceiptList69) EncodeIndex(i int, out *bytes.Buffer) { +// Derivable turns the receipts into a list that can derive the root hash. +func (rl *ReceiptList69) Derivable() types.DerivableList { initBuffers(&rl.buf) - rl.items[i].encodeForHash(rl.buf, out) + return newDerivableRawList(&rl.items, func(data []byte, outbuf *bytes.Buffer) { + var r Receipt + if r.decode69(data) == nil { + r.encodeForHash(rl.buf, outbuf) + } + }) } // DecodeRLP decodes a list receipts from the network format. func (rl *ReceiptList69) DecodeRLP(s *rlp.Stream) error { - if _, err := s.List(); err != nil { - return err - } - for i := 0; s.MoreDataInList(); i++ { - var item Receipt - err := item.decode69(s) - if err != nil { - return fmt.Errorf("receipt %d: %v", i, err) - } - rl.items = append(rl.items, item) - } - return s.ListEnd() + return rl.items.DecodeRLP(s) } // EncodeRLP encodes the list into the network format of eth/69. -func (rl *ReceiptList69) EncodeRLP(_w io.Writer) error { - w := rlp.NewEncoderBuffer(_w) - outer := w.List() - for i := range rl.items { - rl.items[i].encodeForNetwork69(&w) - } - w.ListEnd(outer) - return w.Flush() +func (rl *ReceiptList69) EncodeRLP(w io.Writer) error { + return rl.items.EncodeRLP(w) } // blockReceiptsToNetwork69 takes a slice of rlp-encoded receipts, and transactions, diff --git a/eth/protocols/eth/receipt_test.go b/eth/protocols/eth/receipt_test.go index ea0754ec05..da845f2a3e 100644 --- a/eth/protocols/eth/receipt_test.go +++ b/eth/protocols/eth/receipt_test.go @@ -63,6 +63,18 @@ var receiptsTests = []struct { input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, GasUsedForL1: 1, Status: 1, Logs: receiptsTestLogs2}}, txs: []*types.Transaction{types.NewTx(&types.AccessListTx{})}, }, + { + input: []types.ReceiptForStorage{ + {CumulativeGasUsed: 111, PostState: common.HexToHash("0x1111").Bytes(), Logs: receiptsTestLogs1}, + {CumulativeGasUsed: 222, Status: 0, Logs: receiptsTestLogs2}, + {CumulativeGasUsed: 333, Status: 1, Logs: nil}, + }, + txs: []*types.Transaction{ + types.NewTx(&types.LegacyTx{}), + types.NewTx(&types.AccessListTx{}), + types.NewTx(&types.DynamicFeeTx{}), + }, + }, } func init() { @@ -103,7 +115,10 @@ func TestReceiptList69(t *testing.T) { if err := rlp.DecodeBytes(network, &rl); err != nil { t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) } - rlStorageEnc := rl.EncodeForStorage() + rlStorageEnc, err := rl.EncodeForStorage() + if err != nil { + t.Fatalf("test[%d]: error from EncodeForStorage: %v", i, err) + } if !bytes.Equal(rlStorageEnc, canonDB) { t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB) } @@ -113,7 +128,7 @@ func TestReceiptList69(t *testing.T) { } // compute root hash from ReceiptList69 and compare. - responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil)) + responseHash := types.DeriveSha(rl.Derivable(), trie.NewStackTrie(nil)) if responseHash != test.root { t.Fatalf("test[%d]: wrong root hash from ReceiptList69\nhave: %v\nwant: %v", i, responseHash, test.root) } @@ -140,7 +155,10 @@ func TestReceiptList68(t *testing.T) { if err := rlp.DecodeBytes(network, &rl); err != nil { t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) } - rlStorageEnc := rl.EncodeForStorage() + rlStorageEnc, err := rl.EncodeForStorage() + if err != nil { + t.Fatalf("test[%d]: error from EncodeForStorage: %v", i, err) + } if !bytes.Equal(rlStorageEnc, canonDB) { t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB) } @@ -150,7 +168,7 @@ func TestReceiptList68(t *testing.T) { } // compute root hash from ReceiptList68 and compare. - responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil)) + responseHash := types.DeriveSha(rl.Derivable(), trie.NewStackTrie(nil)) if responseHash != test.root { t.Fatalf("test[%d]: wrong root hash from ReceiptList68\nhave: %v\nwant: %v", i, responseHash, test.root) } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index d69b6951be..f6cbfcf43b 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -31,6 +31,8 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/tracker" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb/database" @@ -94,6 +96,7 @@ func MakeProtocols(backend Backend) []p2p.Protocol { Length: protocolLengths[version], Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { return backend.RunPeer(NewPeer(version, p, rw), func(peer *Peer) error { + defer peer.Close() return Handle(backend, peer) }) }, @@ -147,7 +150,6 @@ func HandleMessage(backend Backend, peer *Peer) error { // Handle the message depending on its contents switch { case msg.Code == GetAccountRangeMsg: - // Decode the account retrieval request var req GetAccountRangePacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -163,23 +165,40 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == AccountRangeMsg: - // A range of accounts arrived to one of our previous requests - res := new(AccountRangePacket) + res := new(accountRangeInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + + // Check response validity. + if len := res.Proof.Len(); len > 128 { + return fmt.Errorf("AccountRange: invalid proof (length %d)", len) + } + tresp := tracker.Response{ID: res.ID, MsgCode: AccountRangeMsg, Size: len(res.Accounts.Content())} + if err := peer.tracker.Fulfil(tresp); err != nil { + return err + } + + // Decode. + accounts, err := res.Accounts.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid accounts list: %v", err) + } + proof, err := res.Proof.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid proof: %v", err) + } + // Ensure the range is monotonically increasing - for i := 1; i < len(res.Accounts); i++ { - if bytes.Compare(res.Accounts[i-1].Hash[:], res.Accounts[i].Hash[:]) >= 0 { - return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, res.Accounts[i-1].Hash[:], i, res.Accounts[i].Hash[:]) + for i := 1; i < len(accounts); i++ { + if bytes.Compare(accounts[i-1].Hash[:], accounts[i].Hash[:]) >= 0 { + return fmt.Errorf("accounts not monotonically increasing: #%d [%x] vs #%d [%x]", i-1, accounts[i-1].Hash[:], i, accounts[i].Hash[:]) } } - requestTracker.Fulfil(peer.id, peer.version, AccountRangeMsg, res.ID) - return backend.Handle(peer, res) + return backend.Handle(peer, &AccountRangePacket{res.ID, accounts, proof}) case msg.Code == GetStorageRangesMsg: - // Decode the storage retrieval request var req GetStorageRangesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -195,25 +214,42 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == StorageRangesMsg: - // A range of storage slots arrived to one of our previous requests - res := new(StorageRangesPacket) + res := new(storageRangesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + + // Check response validity. + if len := res.Proof.Len(); len > 128 { + return fmt.Errorf("StorageRangesMsg: invalid proof (length %d)", len) + } + tresp := tracker.Response{ID: res.ID, MsgCode: StorageRangesMsg, Size: len(res.Slots.Content())} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("StorageRangesMsg: %w", err) + } + + // Decode. + slotLists, err := res.Slots.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid accounts list: %v", err) + } + proof, err := res.Proof.Items() + if err != nil { + return fmt.Errorf("AccountRange: invalid proof: %v", err) + } + // Ensure the ranges are monotonically increasing - for i, slots := range res.Slots { + for i, slots := range slotLists { for j := 1; j < len(slots); j++ { if bytes.Compare(slots[j-1].Hash[:], slots[j].Hash[:]) >= 0 { return fmt.Errorf("storage slots not monotonically increasing for account #%d: #%d [%x] vs #%d [%x]", i, j-1, slots[j-1].Hash[:], j, slots[j].Hash[:]) } } } - requestTracker.Fulfil(peer.id, peer.version, StorageRangesMsg, res.ID) - return backend.Handle(peer, res) + return backend.Handle(peer, &StorageRangesPacket{res.ID, slotLists, proof}) case msg.Code == GetByteCodesMsg: - // Decode bytecode retrieval request var req GetByteCodesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -228,17 +264,25 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == ByteCodesMsg: - // A batch of byte codes arrived to one of our previous requests - res := new(ByteCodesPacket) + res := new(byteCodesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - requestTracker.Fulfil(peer.id, peer.version, ByteCodesMsg, res.ID) - return backend.Handle(peer, res) + length := res.Codes.Len() + tresp := tracker.Response{ID: res.ID, MsgCode: ByteCodesMsg, Size: length} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("ByteCodes: %w", err) + } + + codes, err := res.Codes.Items() + if err != nil { + return fmt.Errorf("ByteCodes: %w", err) + } + + return backend.Handle(peer, &ByteCodesPacket{res.ID, codes}) case msg.Code == GetTrieNodesMsg: - // Decode trie node retrieval request var req GetTrieNodesPacket if err := msg.Decode(&req); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) @@ -255,14 +299,21 @@ func HandleMessage(backend Backend, peer *Peer) error { }) case msg.Code == TrieNodesMsg: - // A batch of trie nodes arrived to one of our previous requests - res := new(TrieNodesPacket) + res := new(trieNodesInput) if err := msg.Decode(res); err != nil { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - requestTracker.Fulfil(peer.id, peer.version, TrieNodesMsg, res.ID) - return backend.Handle(peer, res) + tresp := tracker.Response{ID: res.ID, MsgCode: TrieNodesMsg, Size: res.Nodes.Len()} + if err := peer.tracker.Fulfil(tresp); err != nil { + return fmt.Errorf("TrieNodes: %w", err) + } + nodes, err := res.Nodes.Items() + if err != nil { + return fmt.Errorf("TrieNodes: %w", err) + } + + return backend.Handle(peer, &TrieNodesPacket{res.ID, nodes}) default: return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) @@ -510,21 +561,32 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if reader == nil { reader, _ = triedb.StateReader(req.Root) } + // Retrieve trie nodes until the packet size limit is reached var ( - nodes [][]byte - bytes uint64 - loads int // Trie hash expansions to count database reads + outerIt = req.Paths.ContentIterator() + nodes [][]byte + bytes uint64 + loads int // Trie hash expansions to count database reads ) - for _, pathset := range req.Paths { - switch len(pathset) { + for outerIt.Next() { + innerIt, err := rlp.NewListIterator(outerIt.Value()) + if err != nil { + return nodes, err + } + + switch innerIt.Count() { case 0: // Ensure we penalize invalid requests return nil, fmt.Errorf("%w: zero-item pathset requested", errBadRequest) case 1: // If we're only retrieving an account trie node, fetch it directly - blob, resolved, err := accTrie.GetNode(pathset[0]) + accKey := nextBytes(&innerIt) + if accKey == nil { + return nodes, fmt.Errorf("%w: invalid account node request", errBadRequest) + } + blob, resolved, err := accTrie.GetNode(accKey) loads += resolved // always account database reads, even for failures if err != nil { break @@ -533,33 +595,41 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s bytes += uint64(len(blob)) default: - var stRoot common.Hash - // Storage slots requested, open the storage trie and retrieve from there + accKey := nextBytes(&innerIt) + if accKey == nil { + return nodes, fmt.Errorf("%w: invalid account storage request", errBadRequest) + } + var stRoot common.Hash if reader == nil { // We don't have the requested state snapshotted yet (or it is stale), // but can look up the account via the trie instead. - account, err := accTrie.GetAccountByHash(common.BytesToHash(pathset[0])) + account, err := accTrie.GetAccountByHash(common.BytesToHash(accKey)) loads += 8 // We don't know the exact cost of lookup, this is an estimate if err != nil || account == nil { break } stRoot = account.Root } else { - account, err := reader.Account(common.BytesToHash(pathset[0])) + account, err := reader.Account(common.BytesToHash(accKey)) loads++ // always account database reads, even for failures if err != nil || account == nil { break } stRoot = common.BytesToHash(account.Root) } - id := trie.StorageTrieID(req.Root, common.BytesToHash(pathset[0]), stRoot) + + id := trie.StorageTrieID(req.Root, common.BytesToHash(accKey), stRoot) stTrie, err := trie.NewStateTrie(id, triedb) loads++ // always account database reads, even for failures if err != nil { break } - for _, path := range pathset[1:] { + for innerIt.Next() { + path, _, err := rlp.SplitString(innerIt.Value()) + if err != nil { + return nil, fmt.Errorf("%w: invalid storage key: %v", errBadRequest, err) + } blob, resolved, err := stTrie.GetNode(path) loads += resolved // always account database reads, even for failures if err != nil { @@ -582,6 +652,17 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s return nodes, nil } +func nextBytes(it *rlp.Iterator) []byte { + if !it.Next() { + return nil + } + content, _, err := rlp.SplitString(it.Value()) + if err != nil { + return nil + } + return content +} + // NodeInfo represents a short summary of the `snap` sub-protocol metadata // known about the host peer. type NodeInfo struct{} diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index c57931678c..0b96de4158 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -17,9 +17,13 @@ package snap import ( + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/tracker" + "github.com/ethereum/go-ethereum/rlp" ) // Peer is a collection of relevant information we have about a `snap` peer. @@ -29,6 +33,7 @@ type Peer struct { *p2p.Peer // The embedded P2P package peer rw p2p.MsgReadWriter // Input/output streams for snap version uint // Protocol version negotiated + tracker *tracker.Tracker logger log.Logger // Contextual logger with the peer id injected } @@ -36,22 +41,26 @@ type Peer struct { // NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} id := p.ID().String() return &Peer{ id: id, Peer: p, rw: rw, version: version, + tracker: tracker.New(cap, id, 1*time.Minute), logger: log.New("peer", id[:8]), } } // NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { + cap := p2p.Cap{Name: ProtocolName, Version: version} return &Peer{ id: id, rw: rw, version: version, + tracker: tracker.New(cap, id, 1*time.Minute), logger: log.New("peer", id[:8]), } } @@ -71,63 +80,99 @@ func (p *Peer) Log() log.Logger { return p.logger } +// Close releases resources associated with the peer. +func (p *Peer) Close() { + p.tracker.Stop() +} + // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. -func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes uint64) error { +func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit common.Hash, bytes int) error { p.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetAccountRangeMsg, AccountRangeMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetAccountRangeMsg, + RespCode: AccountRangeMsg, + ID: id, + Size: 2 * bytes, + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetAccountRangeMsg, &GetAccountRangePacket{ ID: id, Root: root, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestStorageRanges fetches a batch of storage slots belonging to one or more // accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. -func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { if len(accounts) == 1 && origin != nil { p.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) } else { p.logger.Trace("Fetching ranges of small storage slots", "reqid", id, "root", root, "accounts", len(accounts), "first", accounts[0], "bytes", common.StorageSize(bytes)) } - requestTracker.Track(p.id, p.version, GetStorageRangesMsg, StorageRangesMsg, id) + + p.tracker.Track(tracker.Request{ + ReqCode: GetStorageRangesMsg, + RespCode: StorageRangesMsg, + ID: id, + Size: 2 * bytes, + }) return p2p.Send(p.rw, GetStorageRangesMsg, &GetStorageRangesPacket{ ID: id, Root: root, Accounts: accounts, Origin: origin, Limit: limit, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestByteCodes fetches a batch of bytecodes by hash. -func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { p.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetByteCodesMsg, ByteCodesMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetByteCodesMsg, + RespCode: ByteCodesMsg, + ID: id, + Size: len(hashes), // ByteCodes is limited by the length of the hash list. + }) + if err != nil { + return err + } return p2p.Send(p.rw, GetByteCodesMsg, &GetByteCodesPacket{ ID: id, Hashes: hashes, - Bytes: bytes, + Bytes: uint64(bytes), }) } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specific state trie. -func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { +// a specific state trie. The `count` is the total count of paths being requested. +func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) - requestTracker.Track(p.id, p.version, GetTrieNodesMsg, TrieNodesMsg, id) + err := p.tracker.Track(tracker.Request{ + ReqCode: GetTrieNodesMsg, + RespCode: TrieNodesMsg, + ID: id, + Size: count, // TrieNodes is limited by number of items. + }) + if err != nil { + return err + } + encPaths, _ := rlp.EncodeToRawList(paths) return p2p.Send(p.rw, GetTrieNodesMsg, &GetTrieNodesPacket{ ID: id, Root: root, - Paths: paths, - Bytes: bytes, + Paths: encPaths, + Bytes: uint64(bytes), }) } diff --git a/eth/protocols/snap/protocol.go b/eth/protocols/snap/protocol.go index 0db206b081..25fe25822b 100644 --- a/eth/protocols/snap/protocol.go +++ b/eth/protocols/snap/protocol.go @@ -78,6 +78,12 @@ type GetAccountRangePacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type accountRangeInput struct { + ID uint64 // ID of the request this is a response for + Accounts rlp.RawList[*AccountData] // List of consecutive accounts from the trie + Proof rlp.RawList[[]byte] // List of trie nodes proving the account range +} + // AccountRangePacket represents an account query response. type AccountRangePacket struct { ID uint64 // ID of the request this is a response for @@ -123,6 +129,12 @@ type GetStorageRangesPacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type storageRangesInput struct { + ID uint64 // ID of the request this is a response for + Slots rlp.RawList[[]*StorageData] // Lists of consecutive storage slots for the requested accounts + Proof rlp.RawList[[]byte] // Merkle proofs for the *last* slot range, if it's incomplete +} + // StorageRangesPacket represents a storage slot query response. type StorageRangesPacket struct { ID uint64 // ID of the request this is a response for @@ -161,6 +173,11 @@ type GetByteCodesPacket struct { Bytes uint64 // Soft limit at which to stop returning data } +type byteCodesInput struct { + ID uint64 // ID of the request this is a response for + Codes rlp.RawList[[]byte] // Requested contract bytecodes +} + // ByteCodesPacket represents a contract bytecode query response. type ByteCodesPacket struct { ID uint64 // ID of the request this is a response for @@ -169,10 +186,10 @@ type ByteCodesPacket struct { // GetTrieNodesPacket represents a state trie node query. type GetTrieNodesPacket struct { - ID uint64 // Request ID to match up responses with - Root common.Hash // Root hash of the account trie to serve - Paths []TrieNodePathSet // Trie node hashes to retrieve the nodes for - Bytes uint64 // Soft limit at which to stop returning data + ID uint64 // Request ID to match up responses with + Root common.Hash // Root hash of the account trie to serve + Paths rlp.RawList[TrieNodePathSet] // Trie node hashes to retrieve the nodes for + Bytes uint64 // Soft limit at which to stop returning data } // TrieNodePathSet is a list of trie node paths to retrieve. A naive way to @@ -187,6 +204,11 @@ type GetTrieNodesPacket struct { // that a slot is accessed before the account path is fully expanded. type TrieNodePathSet [][]byte +type trieNodesInput struct { + ID uint64 // ID of the request this is a response for + Nodes rlp.RawList[[]byte] // Requested state trie nodes +} + // TrieNodesPacket represents a state trie node query response. type TrieNodesPacket struct { ID uint64 // ID of the request this is a response for diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index cf4e494645..08e85c896a 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -412,19 +412,19 @@ type SyncPeer interface { // RequestAccountRange fetches a batch of accounts rooted in a specific account // trie, starting with the origin. - RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error + RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error // RequestStorageRanges fetches a batch of storage slots belonging to one or // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. - RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error + RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error // RequestByteCodes fetches a batch of bytecodes by hash. - RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error + RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in // a specific state trie. - RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error + RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error // Log retrieves the peer's own contextual logger. Log() log.Logger @@ -1102,7 +1102,7 @@ func (s *Syncer) assignAccountTasks(success chan *accountResponse, fail chan *ac if cap < minRequestSize { // Don't bother with peers below a bare minimum performance cap = minRequestSize } - if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, uint64(cap)); err != nil { + if err := peer.RequestAccountRange(reqid, root, req.origin, req.limit, cap); err != nil { peer.Log().Debug("Failed to request account range", "err", err) s.scheduleRevertAccountRequest(req) } @@ -1359,7 +1359,7 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st if subtask != nil { origin, limit = req.origin[:], req.limit[:] } - if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, uint64(cap)); err != nil { + if err := peer.RequestStorageRanges(reqid, root, accounts, origin, limit, cap); err != nil { log.Debug("Failed to request storage", "err", err) s.scheduleRevertStorageRequest(req) } @@ -1492,7 +1492,7 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai defer s.pend.Done() // Attempt to send the remote request and revert if it fails - if err := peer.RequestTrieNodes(reqid, root, pathsets, maxRequestSize); err != nil { + if err := peer.RequestTrieNodes(reqid, root, len(paths), pathsets, maxRequestSize); err != nil { log.Debug("Failed to request trienode healers", "err", err) s.scheduleRevertTrienodeHealRequest(req) } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 97f0349154..0e52e609fe 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -119,10 +119,10 @@ func BenchmarkHashing(b *testing.B) { } type ( - accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error - storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error - trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error - codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error + accountHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error + storageHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error + trieHandlerFunc func(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error + codeHandlerFunc func(t *testPeer, id uint64, hashes []common.Hash, max int) error ) type testPeer struct { @@ -182,21 +182,21 @@ Trienode requests: %d `, t.nAccountRequests, t.nStorageRequests, t.nBytecodeRequests, t.nTrienodeRequests) } -func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error { +func (t *testPeer) RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes int) error { t.logger.Trace("Fetching range of accounts", "reqid", id, "root", root, "origin", origin, "limit", limit, "bytes", common.StorageSize(bytes)) t.nAccountRequests++ go t.accountRequestHandler(t, id, root, origin, limit, bytes) return nil } -func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { +func (t *testPeer) RequestTrieNodes(id uint64, root common.Hash, count int, paths []TrieNodePathSet, bytes int) error { t.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) t.nTrienodeRequests++ go t.trieRequestHandler(t, id, root, paths, bytes) return nil } -func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { +func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes int) error { t.nStorageRequests++ if len(accounts) == 1 && origin != nil { t.logger.Trace("Fetching range of large storage slots", "reqid", id, "root", root, "account", accounts[0], "origin", common.BytesToHash(origin), "limit", common.BytesToHash(limit), "bytes", common.StorageSize(bytes)) @@ -207,7 +207,7 @@ func (t *testPeer) RequestStorageRanges(id uint64, root common.Hash, accounts [] return nil } -func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error { +func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes int) error { t.nBytecodeRequests++ t.logger.Trace("Fetching set of byte codes", "reqid", id, "hashes", len(hashes), "bytes", common.StorageSize(bytes)) go t.codeRequestHandler(t, id, hashes, bytes) @@ -215,7 +215,7 @@ func (t *testPeer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint6 } // defaultTrieRequestHandler is a well-behaving handler for trie healing requests -func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { // Pass the response var nodes [][]byte for _, pathset := range paths { @@ -244,7 +244,7 @@ func defaultTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, } // defaultAccountRequestHandler is a well-behaving handler for AccountRangeRequests -func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { keys, vals, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if err := t.remote.OnAccounts(t, id, keys, vals, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -254,8 +254,8 @@ func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, orig return nil } -func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { - var size uint64 +func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap int) (keys []common.Hash, vals [][]byte, proofs [][]byte) { + var size int if limit == (common.Hash{}) { limit = common.MaxHash } @@ -266,7 +266,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H if bytes.Compare(origin[:], entry.k) <= 0 { keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) } // If we've exceeded the request threshold, abort if bytes.Compare(entry.k, limit[:]) >= 0 { @@ -290,7 +290,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H } // defaultStorageRequestHandler is a well-behaving storage request handler -func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) error { +func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, bOrigin, bLimit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -299,7 +299,7 @@ func defaultStorageRequestHandler(t *testPeer, requestId uint64, root common.Has return nil } -func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes { bytecodes = append(bytecodes, getCodeByHash(h)) @@ -311,8 +311,8 @@ func defaultCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max return nil } -func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { - var size uint64 +func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size int for _, account := range accounts { // The first account might start from a different origin and end sooner var originHash common.Hash @@ -338,7 +338,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm } keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) if bytes.Compare(entry.k, limitHash[:]) >= 0 { break } @@ -377,8 +377,8 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm // createStorageRequestResponseAlwaysProve tests a cornercase, where the peer always // supplies the proof for the last account, even if it is 'complete'. -func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { - var size uint64 +func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max int) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { + var size int max = max * 3 / 4 var origin common.Hash @@ -395,7 +395,7 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco } keys = append(keys, common.BytesToHash(entry.k)) vals = append(vals, entry.v) - size += uint64(32 + len(entry.v)) + size += 32 + len(entry.v) if size > max { exit = true } @@ -433,34 +433,34 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco } // emptyRequestAccountRangeFn is a rejects AccountRangeRequests -func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func emptyRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { t.remote.OnAccounts(t, requestId, nil, nil, nil) return nil } -func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func nonResponsiveRequestAccountRangeFn(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { return nil } -func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func emptyTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { t.remote.OnTrieNodes(t, requestId, nil) return nil } -func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap uint64) error { +func nonResponsiveTrieRequestHandler(t *testPeer, requestId uint64, root common.Hash, paths []TrieNodePathSet, cap int) error { return nil } -func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func emptyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { t.remote.OnStorage(t, requestId, nil, nil, nil) return nil } -func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func nonResponsiveStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return nil } -func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponseAlwaysProve(t, root, accounts, origin, limit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, proofs); err != nil { t.test.Errorf("Remote side rejected our delivery: %v", err) @@ -475,7 +475,7 @@ func proofHappyStorageRequestHandler(t *testPeer, requestId uint64, root common. // return nil //} -func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes { // Send back the hashes @@ -489,7 +489,7 @@ func corruptCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max return nil } -func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { +func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max int) error { var bytecodes [][]byte for _, h := range hashes[:1] { bytecodes = append(bytecodes, getCodeByHash(h)) @@ -503,11 +503,11 @@ func cappedCodeRequestHandler(t *testPeer, id uint64, hashes []common.Hash, max } // starvingStorageRequestHandler is somewhat well-behaving storage handler, but it caps the returned results to be very small -func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func starvingStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return defaultStorageRequestHandler(t, requestId, root, accounts, origin, limit, 500) } -func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { return defaultAccountRequestHandler(t, requestId, root, origin, limit, 500) } @@ -515,7 +515,7 @@ func starvingAccountRequestHandler(t *testPeer, requestId uint64, root common.Ha // return defaultAccountRequestHandler(t, requestId-1, root, origin, 500) //} -func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { +func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { hashes, accounts, proofs := createAccountRequestResponse(t, root, origin, limit, cap) if len(proofs) > 0 { proofs = proofs[1:] @@ -529,7 +529,7 @@ func corruptAccountRequestHandler(t *testPeer, requestId uint64, root common.Has } // corruptStorageRequestHandler doesn't provide good proofs -func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, proofs := createStorageRequestResponse(t, root, accounts, origin, limit, max) if len(proofs) > 0 { proofs = proofs[1:] @@ -542,7 +542,7 @@ func corruptStorageRequestHandler(t *testPeer, requestId uint64, root common.Has return nil } -func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { +func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { hashes, slots, _ := createStorageRequestResponse(t, root, accounts, origin, limit, max) if err := t.remote.OnStorage(t, requestId, hashes, slots, nil); err != nil { t.logger.Info("remote error on delivery (as expected)", "error", err) @@ -577,7 +577,7 @@ func testSyncBloatedProof(t *testing.T, scheme string) { source.accountTrie = sourceAccountTrie.Copy() source.accountValues = elems - source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) error { + source.accountRequestHandler = func(t *testPeer, requestId uint64, root common.Hash, origin common.Hash, limit common.Hash, cap int) error { var ( keys []common.Hash vals [][]byte @@ -1165,7 +1165,7 @@ func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) { var counter int syncer := setupSyncer( nodeScheme, - mkSource("capped", func(t *testPeer, id uint64, hashes []common.Hash, max uint64) error { + mkSource("capped", func(t *testPeer, id uint64, hashes []common.Hash, max int) error { counter++ return cappedCodeRequestHandler(t, id, hashes, max) }), @@ -1432,7 +1432,7 @@ func testSyncWithUnevenStorage(t *testing.T, scheme string) { source.accountValues = accounts source.setStorageTries(storageTries) source.storageValues = storageElems - source.storageRequestHandler = func(t *testPeer, reqId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + source.storageRequestHandler = func(t *testPeer, reqId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max int) error { return defaultStorageRequestHandler(t, reqId, root, accounts, origin, limit, 128) // retrieve storage in large mode } return source diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cf02cbe254..bcfe7ba2e7 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -157,7 +157,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u if current = eth.blockchain.GetBlockByNumber(next); current == nil { return nil, nil, fmt.Errorf("block #%d not found", next) } - _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) + _, err := eth.blockchain.Processor().Process(ctx, current, statedb, vm.Config{}) if err != nil { return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } diff --git a/eth/sync_test.go b/eth/sync_test.go index dc295f2790..509b836f82 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -36,17 +36,11 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { t.Parallel() // Create an empty handler and ensure it's in snap sync mode - empty := newTestHandler() - if !empty.handler.snapSync.Load() { - t.Fatalf("snap sync disabled on pristine blockchain") - } + empty := newTestHandler(ethconfig.SnapSync) defer empty.close() // Create a full handler and ensure snap sync ends up disabled - full := newTestHandlerWithBlocks(1024) - if full.handler.snapSync.Load() { - t.Fatalf("snap sync not disabled on non-empty blockchain") - } + full := newTestHandlerWithBlocks(1024, ethconfig.SnapSync) defer full.close() // Sync up the two handlers via both `eth` and `snap` @@ -85,7 +79,7 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - if err := empty.handler.downloader.BeaconSync(ethconfig.SnapSync, full.chain.CurrentBlock(), nil); err != nil { + if err := empty.handler.downloader.BeaconSync(full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } // Downloader internally has to wait for a timer (3s) to be expired before @@ -96,7 +90,7 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { case <-timeout: t.Fatalf("snap sync not disabled after successful synchronisation") case <-time.After(100 * time.Millisecond): - if !empty.handler.snapSync.Load() { + if empty.handler.downloader.ConfigSyncMode() == ethconfig.FullSync { return } } diff --git a/eth/syncer/syncer.go b/eth/syncer/syncer.go index 6b33ec54ba..c0d54b953b 100644 --- a/eth/syncer/syncer.go +++ b/eth/syncer/syncer.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" @@ -129,17 +130,43 @@ func (s *Syncer) run() { break } if resync { - req.errc <- s.backend.Downloader().BeaconDevSync(ethconfig.FullSync, target) + if mode := s.backend.Downloader().ConfigSyncMode(); mode != ethconfig.FullSync { + req.errc <- fmt.Errorf("unsupported syncmode %v, please relaunch geth with --syncmode full", mode) + } else { + req.errc <- s.backend.Downloader().BeaconDevSync(target) + } } case <-ticker.C: - if target == nil || !s.exitWhenSynced { + if target == nil { continue } - if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { - log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) - go s.stack.Close() // async since we need to close ourselves - return + + // Terminate the node if the target has been reached + if s.exitWhenSynced { + if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { + log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) + go s.stack.Close() // async since we need to close ourselves + return + } + } + + // Set the finalized and safe markers relative to the current head. + // The finalized marker is set two epochs behind the target, + // and the safe marker is set one epoch behind the target. + head := s.backend.BlockChain().CurrentHeader() + if head == nil { + continue + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength*2); header != nil { + if final := s.backend.BlockChain().CurrentFinalBlock(); final == nil || final.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetFinalized(header) + } + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength); header != nil { + if safe := s.backend.BlockChain().CurrentSafeBlock(); safe == nil || safe.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetSafe(header) + } } case <-s.closed: diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 814e13a200..2b556ce9a8 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -1000,7 +1000,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc } var ( msg = args.ToMessage(blockContext.BaseFee, api.backend.RPCGasCap(), block.Header(), statedb, core.NewMessageEthcallContext(), true) - tx = args.ToTransaction(types.LegacyTxType) + tx = args.ToTransaction(types.DynamicFeeTxType) traceConfig *TraceConfig ) // Lower the basefee to 0 to avoid breaking EVM diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 80653b4fab..022b9aadf1 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -1379,3 +1379,350 @@ func TestStandardTraceBlockToFile(t *testing.T) { } } } + +func TestTraceBadBlock(t *testing.T) { + t.Parallel() + + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + txHashs = make([]common.Hash, 0, 2) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.PUSH1), 0x2a, // push 42 + byte(vm.PUSH1), 0x00, // push slot 0 + byte(vm.SSTORE), // sstore(0, 42) + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + // tx 1: call storage contract (executes PUSH1, PUSH1, SSTORE, STOP) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block so parent state is available + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + api := NewAPI(backend) + result, err := api.TraceBadBlock(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 tx traces, got %d", len(result)) + } + + // First tx: plain transfer + have, _ := json.Marshal(result) + var traces []struct { + TxHash common.Hash `json:"txHash"` + Result struct { + Gas uint64 `json:"gas"` + Failed bool `json:"failed"` + StructLogs []json.RawMessage `json:"structLogs"` + } `json:"result"` + } + if err := json.Unmarshal(have, &traces); err != nil { + t.Fatalf("failed to unmarshal traces: %v", err) + } + if traces[0].TxHash != txHashs[0] { + t.Errorf("tx 0: hash mismatch, have %v, want %v", traces[0].TxHash, txHashs[0]) + } + if traces[0].Result.Gas != params.TxGas { + t.Errorf("tx 0: gas mismatch, have %d, want %d", traces[0].Result.Gas, params.TxGas) + } + if len(traces[0].Result.StructLogs) != 0 { + t.Errorf("tx 0: expected empty structLogs for plain transfer, got %d entries", len(traces[0].Result.StructLogs)) + } + + // Second tx: contract call + if traces[1].TxHash != txHashs[1] { + t.Errorf("tx 1: hash mismatch, have %v, want %v", traces[1].TxHash, txHashs[1]) + } + if traces[1].Result.Failed { + t.Error("tx 1: expected success, got failed") + } + // Contract has 4 opcodes: PUSH1, PUSH1, SSTORE, STOP + if len(traces[1].Result.StructLogs) != 4 { + t.Errorf("tx 1: expected 4 structLog entries for contract call, got %d", len(traces[1].Result.StructLogs)) + } + + // Non-existent bad block + _, err = api.TraceBadBlock(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } + wantErr := fmt.Sprintf("bad block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestIntermediateRoots(t *testing.T) { + t.Parallel() + + // Initialize test accounts and a contract that writes to storage. + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + // Contract: SSTORE(CALLVALUE, CALLVALUE) + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.CALLVALUE), + byte(vm.CALLVALUE), + byte(vm.SSTORE), + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 1: sstore(1, 1) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(1), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 2: sstore(2, 2) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 2, + To: &storageContract, + Value: big.NewInt(2), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + }) + defer backend.teardown() + + api := NewAPI(backend) + block := backend.chain.GetBlockByNumber(1) + + // Should return one root per tx + roots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(roots) != 3 { + t.Fatalf("root count mismatch, have %d, want 3", len(roots)) + } + for i, root := range roots { + if root == (common.Hash{}) { + t.Errorf("root[%d] should not be zero", i) + } + } + if roots[0] == roots[1] { + t.Error("root[0] and root[1] should differ (transfer vs sstore)") + } + if roots[1] == roots[2] { + t.Error("root[1] and root[2] should differ (sstore to different slots)") + } + + // Intermediate roots of a bad block + rawdb.WriteBadBlock(backend.chaindb, block) + badRoots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error for bad block fallback, have %v", err) + } + if !reflect.DeepEqual(roots, badRoots) { + t.Errorf("bad block roots mismatch, have %v, want %v", badRoots, roots) + } + + // Genesis block: should return error + genesisBlock := backend.chain.GetBlockByNumber(0) + _, err = api.IntermediateRoots(context.Background(), genesisBlock.Hash(), nil) + if err == nil || err.Error() != "genesis is not traceable" { + t.Fatalf("want 'genesis is not traceable' error, have %v", err) + } + + // Non-existent block: should return error + _, err = api.IntermediateRoots(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent block, have none") + } + wantErr := fmt.Sprintf("block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestStandardTraceBadBlockToFile(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + + aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43") + aaCode = []byte{byte(vm.PUSH1), 0x00, byte(vm.POP)} + + bb = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f44") + bbCode = []byte{byte(vm.PUSH2), 0x00, 0x01, byte(vm.POP)} + ) + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + aa: { + Code: aaCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + bb: { + Code: bbCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + }, + } + txHashs := make([]common.Hash, 0, 2) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &bb, + Value: big.NewInt(1), + Gas: 100000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + var testSuite = []struct { + config *StdTraceConfig + want []string + }{ + { + // All txs traced + config: nil, + want: []string{ + `{"pc":0,"op":96,"gas":"0x7148","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"} +{"pc":2,"op":80,"gas":"0x7145","gasCost":"0x2","memSize":0,"stack":["0x0"],"depth":1,"refund":0,"opName":"POP"} +{"pc":3,"op":0,"gas":"0x7143","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + { + // Specific tx traced + config: &StdTraceConfig{TxHash: txHashs[1]}, + want: []string{ + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + } + + api := NewAPI(backend) + for i, tc := range testSuite { + txTraces, err := api.StandardTraceBadBlockToFile(context.Background(), block.Hash(), tc.config) + if err != nil { + t.Fatalf("test %d: unexpected error %v", i, err) + } + if len(txTraces) != len(tc.want) { + t.Fatalf("test %d: file count mismatch, have %d, want %d", i, len(txTraces), len(tc.want)) + } + for j, traceFileName := range txTraces { + defer os.Remove(traceFileName) + traceReceived, err := os.ReadFile(traceFileName) + if err != nil { + t.Fatalf("test %d: could not read trace file: %v", i, err) + } + if tc.want[j] != string(traceReceived) { + t.Fatalf("test %d, trace %d: result mismatch\nhave:\n%s\nwant:\n%s", i, j, string(traceReceived), tc.want[j]) + } + } + } + + // Non-existent bad block + _, err := api.StandardTraceBadBlockToFile(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } +} diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 1bca0ea3eb..dadc5c4582 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -43,6 +43,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` Position hexutil.Uint `json:"position"` } @@ -311,7 +312,7 @@ func TestInternals(t *testing.T) { byte(vm.LOG0), }, tracer: mkTracer("callTracer", json.RawMessage(`{ "withLog": true }`)), - want: fmt.Sprintf(`{"beforeEVMTransfers":[{"purpose":"feePayment","from":"%s","to":null,"value":"0x13880"}],"afterEVMTransfers":[{"purpose":"gasRefund","from":null,"to":"%s","value":"0xdce2"},{"purpose":"tip","from":null,"to":"0x0000000000000000000000000000000000000000","value":"0x5b9e"}],"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","position":"0x0"}],"value":"0x0","type":"CALL"}`, origin.Hex(), origin.Hex(), originHex), + want: fmt.Sprintf(`{"beforeEVMTransfers":[{"purpose":"feePayment","from":"%s","to":null,"value":"0x13880"}],"afterEVMTransfers":[{"purpose":"gasRefund","from":null,"to":"%s","value":"0xdce2"},{"purpose":"tip","from":null,"to":"0x0000000000000000000000000000000000000000","value":"0x5b9e"}],"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","index":"0x0","position":"0x0"}],"value":"0x0","type":"CALL"}`, origin.Hex(), origin.Hex(), originHex), }, { // Leads to OOM on the prestate tracer diff --git a/eth/tracers/internal/tracetest/selfdestruct_state_test.go b/eth/tracers/internal/tracetest/selfdestruct_state_test.go new file mode 100644 index 0000000000..9e4a1dda2b --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_state_test.go @@ -0,0 +1,653 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracetest + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// accountState represents the expected final state of an account +type accountState struct { + Balance *big.Int + Nonce uint64 + Code []byte + Exists bool +} + +// selfdestructStateTracer tracks state changes during selfdestruct operations +type selfdestructStateTracer struct { + env *tracing.VMContext + accounts map[common.Address]*accountState +} + +func newSelfdestructStateTracer() *selfdestructStateTracer { + return &selfdestructStateTracer{ + accounts: make(map[common.Address]*accountState), + } +} + +func (t *selfdestructStateTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) { + t.env = env +} + +func (t *selfdestructStateTracer) OnTxEnd(receipt *types.Receipt, err error) { + // Nothing to do +} + +func (t *selfdestructStateTracer) getOrCreateAccount(addr common.Address) *accountState { + if acc, ok := t.accounts[addr]; ok { + return acc + } + + // Initialize with current state from statedb + acc := &accountState{ + Balance: t.env.StateDB.GetBalance(addr).ToBig(), + Nonce: t.env.StateDB.GetNonce(addr), + Code: t.env.StateDB.GetCode(addr), + Exists: t.env.StateDB.Exist(addr), + } + t.accounts[addr] = acc + return acc +} + +func (t *selfdestructStateTracer) OnBalanceChange(addr common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Balance = new +} + +func (t *selfdestructStateTracer) OnNonceChangeV2(addr common.Address, prev, new uint64, reason tracing.NonceChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Nonce = new + + // If this is a selfdestruct nonce change, mark account as not existing + if reason == tracing.NonceChangeSelfdestruct { + acc.Exists = false + } +} + +func (t *selfdestructStateTracer) OnCodeChangeV2(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + acc := t.getOrCreateAccount(addr) + acc.Code = code + + // If this is a selfdestruct code change, mark account as not existing + if reason == tracing.CodeChangeSelfDestruct { + acc.Exists = false + } +} + +func (t *selfdestructStateTracer) Hooks() *tracing.Hooks { + return &tracing.Hooks{ + OnTxStart: t.OnTxStart, + OnTxEnd: t.OnTxEnd, + OnBalanceChange: t.OnBalanceChange, + OnNonceChangeV2: t.OnNonceChangeV2, + OnCodeChangeV2: t.OnCodeChangeV2, + } +} + +func (t *selfdestructStateTracer) Accounts() map[common.Address]*accountState { + return t.accounts +} + +// verifyAccountState compares actual and expected account state and reports any mismatches +func verifyAccountState(t *testing.T, addr common.Address, actual, expected *accountState) { + if actual.Balance.Cmp(expected.Balance) != 0 { + t.Errorf("address %s: balance mismatch: have %s, want %s", + addr.Hex(), actual.Balance, expected.Balance) + } + if actual.Nonce != expected.Nonce { + t.Errorf("address %s: nonce mismatch: have %d, want %d", + addr.Hex(), actual.Nonce, expected.Nonce) + } + if len(actual.Code) != len(expected.Code) { + t.Errorf("address %s: code length mismatch: have %d, want %d", + addr.Hex(), len(actual.Code), len(expected.Code)) + } + if actual.Exists != expected.Exists { + t.Errorf("address %s: exists mismatch: have %v, want %v", + addr.Hex(), actual.Exists, expected.Exists) + } +} + +// setupTestBlockchain creates a blockchain with the given genesis and transaction, +// returns the blockchain, the first block, and a statedb at genesis for testing +func setupTestBlockchain(t *testing.T, genesis *core.Genesis, tx *types.Transaction, useBeacon bool) (*core.BlockChain, *types.Block, *state.StateDB) { + var engine consensus.Engine + if useBeacon { + engine = beacon.New(ethash.NewFaker()) + } else { + engine = ethash.NewFaker() + } + + _, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, 1, func(i int, b *core.BlockGen) { + b.AddTx(tx) + }) + db := rawdb.NewMemoryDatabase() + blockchain, err := core.NewBlockChain(db, nil, genesis, engine, nil) + if err != nil { + t.Fatalf("failed to create blockchain: %v", err) + } + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("failed to insert chain: %v", err) + } + genesisBlock := blockchain.GetBlockByNumber(0) + if genesisBlock == nil { + t.Fatalf("failed to get genesis block") + } + statedb, err := blockchain.StateAt(genesisBlock.Root()) + if err != nil { + t.Fatalf("failed to get state: %v", err) + } + + return blockchain, blocks[0], statedb +} + +func TestSelfdestructStateTracer(t *testing.T) { + t.Parallel() + + const ( + // Gas limit high enough for all test scenarios (factory creation + multiple calls) + testGasLimit = 500000 + + // Common balance amounts used across tests + testBalanceInitial = 100 // Initial balance for contracts being tested + testBalanceSent = 50 // Amount sent back in sendback tests + testBalanceFactory = 200 // Factory needs extra balance for contract creation + ) + + // Helper to create *big.Int for wei amounts + wei := func(amount int64) *big.Int { + return big.NewInt(amount) + } + + // Test account (transaction sender) + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + caller = crypto.PubkeyToAddress(key.PublicKey) + ) + + // Simple selfdestruct test contracts + var ( + contract = common.HexToAddress("0x00000000000000000000000000000000000000bb") + recipient = common.HexToAddress("0x00000000000000000000000000000000000000cc") + ) + // Build selfdestruct code: PUSH20 SELFDESTRUCT + selfdestructCode := []byte{byte(vm.PUSH20)} + selfdestructCode = append(selfdestructCode, recipient.Bytes()...) + selfdestructCode = append(selfdestructCode, byte(vm.SELFDESTRUCT)) + + // Factory test contracts (create-and-destroy pattern) + var ( + factory = common.HexToAddress("0x00000000000000000000000000000000000000ff") + ) + // Factory code: creates a contract with 100 wei and calls it to trigger selfdestruct back to factory + // See selfdestruct_test_contracts/factory.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factory.yul --bin + // (Using paris to avoid PUSH0 opcode which is not available pre-Shanghai) + var ( + factoryCode = common.Hex2Bytes("6a6133ff6000526002601ef360a81b600052600080808080600b816064f05af100") + createdContractAddr = crypto.CreateAddress(factory, 0) // Address where factory creates the contract + ) + + // Sendback test contracts (A→B→A pattern) + // For the refund test: Coordinator calls A, then B + // A selfdestructs to B, B sends funds back to A + var ( + contractA = common.HexToAddress("0x00000000000000000000000000000000000000aa") + contractB = common.HexToAddress("0x00000000000000000000000000000000000000bb") + coordinator = common.HexToAddress("0x00000000000000000000000000000000000000cc") + ) + // Contract A: if msg.value > 0, accept funds; else selfdestruct to B + // See selfdestruct_test_contracts/contractA.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractA.yul --bin + contractACode := common.Hex2Bytes("60003411600a5760bbff5b00") + + // Contract B: sends 50 wei back to contract A + // See selfdestruct_test_contracts/contractB.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractB.yul --bin + contractBCode := common.Hex2Bytes("6000808080603260aa5af100") + + // Coordinator: calls A (A selfdestructs to B), then calls B (B sends funds to A) + // See selfdestruct_test_contracts/coordinator.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris coordinator.yul --bin + coordinatorCode := common.Hex2Bytes("60008080808060aa818080808060bb955af1505af100") + + // Factory for create-and-refund test: creates A with 100 wei, calls A, calls B + // See selfdestruct_test_contracts/factoryRefund.yul for source + // Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factoryRefund.yul --bin + var ( + factoryRefund = common.HexToAddress("0x00000000000000000000000000000000000000dd") + factoryRefundCode = common.Hex2Bytes("60008080808060bb78600c600d600039600c6000f3fe60003411600a5760bbff5b0082528180808080601960076064f05af1505af100") + createdContractAddrA = crypto.CreateAddress(factoryRefund, 0) // Address where factory creates contract A + ) + + // Self-destruct-to-self test contracts + var ( + contractSelfDestruct = common.HexToAddress("0x00000000000000000000000000000000000000aa") + coordinatorSendAfter = common.HexToAddress("0x00000000000000000000000000000000000000ee") + ) + // Contract that selfdestructs to self + // See selfdestruct_test_contracts/contractSelfDestruct.yul + contractSelfDestructCode := common.Hex2Bytes("30ff") + + // Coordinator: calls contract (triggers selfdestruct to self), stores balance, sends 50 wei, stores balance again + // See selfdestruct_test_contracts/coordinatorSendAfter.yul + coordinatorSendAfterCode := common.Hex2Bytes("60aa600080808080855af150803160005560008080806032855af1503160015500") + + // Factory with balance checking: creates contract, calls it, checks balances + // See selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul + var ( + factorySelfDestructBalanceCheck = common.HexToAddress("0x00000000000000000000000000000000000000fd") + factorySelfDestructBalanceCheckCode = common.Hex2Bytes("6e6002600d60003960026000f3fe30ff600052600f60116064f0600080808080855af150803160005560008080806032855af1503160015500") + createdContractAddrSelfBalanceCheck = crypto.CreateAddress(factorySelfDestructBalanceCheck, 0) + ) + + tests := []struct { + name string + description string + targetContract common.Address + genesis *core.Genesis + useBeacon bool + expectedResults map[common.Address]accountState + expectedStorage map[common.Address]map[uint64]*big.Int + }{ + { + name: "pre_6780_existing", + description: "Pre-EIP-6780: Existing contract selfdestructs to recipient. Contract should be destroyed and balance transferred.", + targetContract: contract, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contract: { + Balance: wei(testBalanceInitial), + Code: selfdestructCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + contract: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + recipient: { + Balance: wei(testBalanceInitial), // Received contract's balance + Nonce: 0, + Code: []byte{}, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing", + description: "Post-EIP-6780: Existing contract selfdestructs to recipient. Balance transferred but contract NOT destroyed (code/storage remain).", + targetContract: contract, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contract: { + Balance: wei(testBalanceInitial), + Code: selfdestructCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contract: { + Balance: wei(0), + Nonce: 0, + Code: selfdestructCode, + Exists: true, + }, + recipient: { + Balance: wei(testBalanceInitial), + Nonce: 0, + Code: []byte{}, + Exists: true, + }, + }, + }, + { + name: "pre_6780_create_destroy", + description: "Pre-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed, factory gets refund.", + targetContract: factory, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factory: { + Balance: wei(testBalanceFactory), + Code: factoryCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + factory: { + Balance: wei(testBalanceFactory), + Nonce: 1, + Code: factoryCode, + Exists: true, + }, + createdContractAddr: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + }, + }, + { + name: "post_6780_create_destroy", + description: "Post-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed (EIP-6780 exception for same-tx creation).", + targetContract: factory, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factory: { + Balance: wei(testBalanceFactory), + Code: factoryCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + factory: { + Balance: wei(testBalanceFactory), + Nonce: 1, + Code: factoryCode, + Exists: true, + }, + createdContractAddr: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + }, + }, + { + name: "pre_6780_sendback", + description: "Pre-EIP-6780: Contract A selfdestructs sending funds to B, then B sends funds back to A's address. Funds sent to destroyed address are burnt.", + targetContract: coordinator, + genesis: &core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractA: { + Balance: wei(testBalanceInitial), + Code: contractACode, + }, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + coordinator: { + Code: coordinatorCode, + }, + }, + }, + useBeacon: false, + expectedResults: map[common.Address]accountState{ + contractA: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + contractB: { + // 100 received - 50 sent back + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing_sendback", + description: "Post-EIP-6780: Existing contract A selfdestructs to B, then B sends funds back to A. Funds are NOT burnt (A still exists post-6780).", + targetContract: coordinator, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractA: { + Balance: wei(testBalanceInitial), + Code: contractACode, + }, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + coordinator: { + Code: coordinatorCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contractA: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractACode, + Exists: true, + }, + contractB: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_create_destroy_sendback", + description: "Post-EIP-6780: Factory creates A, A selfdestructs to B, B sends funds back to A. Funds are burnt (A was destroyed via EIP-6780 exception).", + targetContract: factoryRefund, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractB: { + Balance: wei(0), + Code: contractBCode, + }, + factoryRefund: { + Balance: wei(testBalanceFactory), + Code: factoryRefundCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + createdContractAddrA: { + // Funds sent back are burnt! + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + contractB: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: contractBCode, + Exists: true, + }, + }, + }, + { + name: "post_6780_existing_to_self", + description: "Post-EIP-6780: Pre-existing contract selfdestructs to itself. Balance NOT burnt (selfdestruct-to-self is no-op for existing contracts).", + targetContract: coordinatorSendAfter, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + contractSelfDestruct: { + Balance: wei(testBalanceInitial), + Code: contractSelfDestructCode, + }, + coordinatorSendAfter: { + Balance: wei(testBalanceInitial), + Code: coordinatorSendAfterCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + contractSelfDestruct: { + Balance: wei(150), + Nonce: 0, + Code: contractSelfDestructCode, + Exists: true, + }, + coordinatorSendAfter: { + Balance: wei(testBalanceSent), + Nonce: 0, + Code: coordinatorSendAfterCode, + Exists: true, + }, + }, + expectedStorage: map[common.Address]map[uint64]*big.Int{ + coordinatorSendAfter: { + 0: wei(testBalanceInitial), + 1: wei(150), + }, + }, + }, + { + name: "post_6780_create_destroy_to_self", + description: "Post-EIP-6780: Factory creates contract, contract selfdestructs to itself. Balance IS burnt and contract destroyed (EIP-6780 exception for same-tx creation).", + targetContract: factorySelfDestructBalanceCheck, + genesis: &core.Genesis{ + Config: params.AllDevChainProtocolChanges, + Alloc: types.GenesisAlloc{ + caller: {Balance: big.NewInt(params.Ether)}, + factorySelfDestructBalanceCheck: { + Balance: wei(testBalanceFactory), + Code: factorySelfDestructBalanceCheckCode, + }, + }, + }, + useBeacon: true, + expectedResults: map[common.Address]accountState{ + createdContractAddrSelfBalanceCheck: { + Balance: wei(0), + Nonce: 0, + Code: []byte{}, + Exists: false, + }, + factorySelfDestructBalanceCheck: { + Balance: wei(testBalanceSent), + Nonce: 1, + Code: factorySelfDestructBalanceCheckCode, + Exists: true, + }, + }, + expectedStorage: map[common.Address]map[uint64]*big.Int{ + factorySelfDestructBalanceCheck: { + 0: wei(0), + 1: wei(0), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + signer = types.HomesteadSigner{} + tx *types.Transaction + err error + ) + + tx, err = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &tt.targetContract, + Value: big.NewInt(0), + Gas: testGasLimit, + GasPrice: big.NewInt(params.InitialBaseFee * 2), + Data: nil, + }), signer, key) + if err != nil { + t.Fatalf("failed to sign transaction: %v", err) + } + + blockchain, block, statedb := setupTestBlockchain(t, tt.genesis, tx, tt.useBeacon) + defer blockchain.Stop() + + tracer := newSelfdestructStateTracer() + hookedState := state.NewHookedState(statedb, tracer.Hooks()) + msg, err := core.TransactionToMessage(tx, signer, nil, core.NewMessageReplayContext()) + if err != nil { + t.Fatalf("failed to prepare transaction for tracing: %v", err) + } + context := core.NewEVMBlockContext(block.Header(), blockchain, nil) + evm := vm.NewEVM(context, hookedState, tt.genesis.Config, vm.Config{Tracer: tracer.Hooks()}) + usedGas := uint64(0) + _, _, err = core.ApplyTransactionWithEVM(msg, new(core.GasPool).AddGas(tx.Gas()), statedb, block.Number(), block.Hash(), block.Time(), tx, &usedGas, evm, nil) + if err != nil { + t.Fatalf("failed to execute transaction: %v", err) + } + + results := tracer.Accounts() + + // Verify storage + for addr, expectedSlots := range tt.expectedStorage { + for slot, expectedValue := range expectedSlots { + actualValue := statedb.GetState(addr, common.BigToHash(big.NewInt(int64(slot)))) + if actualValue.Big().Cmp(expectedValue) != 0 { + t.Errorf("address %s slot %d: storage mismatch: have %s, want %s", + addr.Hex(), slot, actualValue.Big(), expectedValue) + } + } + } + + // Verify results + for addr, expected := range tt.expectedResults { + actual, ok := results[addr] + if !ok { + t.Errorf("address %s missing from results", addr.Hex()) + continue + } + verifyAccountState(t, addr, actual, &expected) + } + }) + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul new file mode 100644 index 0000000000..109551f26e --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul @@ -0,0 +1,18 @@ +object "ContractA" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // If receiving funds (msg.value > 0), just accept them and return + if gt(callvalue(), 0) { + stop() + } + + // Otherwise, selfdestruct to B (transfers balance immediately, then stops execution) + let contractB := 0x00000000000000000000000000000000000000bb + selfdestruct(contractB) + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul new file mode 100644 index 0000000000..c737355fb6 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul @@ -0,0 +1,14 @@ +object "ContractB" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Send 50 wei back to contract A + let contractA := 0x00000000000000000000000000000000000000aa + let success := call(gas(), contractA, 50, 0, 0, 0, 0) + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul new file mode 100644 index 0000000000..73884c5dd4 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul @@ -0,0 +1,12 @@ +object "ContractSelfDestruct" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Simply selfdestruct to self + selfdestruct(address()) + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul new file mode 100644 index 0000000000..54bd5c08f3 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul @@ -0,0 +1,20 @@ +object "Coordinator" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractA := 0x00000000000000000000000000000000000000aa + let contractB := 0x00000000000000000000000000000000000000bb + + // First, call A (A will selfdestruct to B) + pop(call(gas(), contractA, 0, 0, 0, 0, 0)) + + // Then, call B (B will send funds back to A) + pop(call(gas(), contractB, 0, 0, 0, 0, 0)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul new file mode 100644 index 0000000000..9473d1f3ef --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul @@ -0,0 +1,27 @@ +object "CoordinatorSendAfter" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractAddr := 0x00000000000000000000000000000000000000aa + + // Call contract (triggers selfdestruct to self, burning its balance) + pop(call(gas(), contractAddr, 0, 0, 0, 0, 0)) + + // Check contract's balance immediately after selfdestruct + // Store in slot 0 to verify it's 0 (proving immediate burn) + sstore(0, balance(contractAddr)) + + // Send 50 wei to the contract (after it selfdestructed) + pop(call(gas(), contractAddr, 50, 0, 0, 0, 0)) + + // Check balance again after sending funds + // Store in slot 1 to verify it's 50 (new funds not burnt) + sstore(1, balance(contractAddr)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul new file mode 100644 index 0000000000..f52a46fcc3 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul @@ -0,0 +1,28 @@ +object "FactoryRefund" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + let contractB := 0x00000000000000000000000000000000000000bb + + // Store the deploy bytecode for contract A in memory + // Full deploy bytecode from: solc --strict-assembly --evm-version paris contractA.yul --bin + // Including the 0xfe separator: 600c600d600039600c6000f3fe60003411600a5760bbff5b00 + // That's 25 bytes, padded to 32 bytes with 7 zero bytes at the front + mstore(0, 0x0000000000000000000000000000600c600d600039600c6000f3fe60003411600a5760bbff5b00) + + // CREATE contract A with 100 wei, using 25 bytes starting at position 7 + let contractA := create(100, 7, 25) + + // Call contract A (triggers selfdestruct to B) + pop(call(gas(), contractA, 0, 0, 0, 0, 0)) + + // Call contract B (B sends 50 wei back to A) + pop(call(gas(), contractB, 0, 0, 0, 0, 0)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul new file mode 100644 index 0000000000..46f4628419 --- /dev/null +++ b/eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul @@ -0,0 +1,35 @@ +object "FactorySelfDestructBalanceCheck" { + code { + datacopy(0, dataoffset("Runtime"), datasize("Runtime")) + return(0, datasize("Runtime")) + } + object "Runtime" { + code { + // Get the full deploy bytecode for ContractSelfDestruct + // Compiled with: solc --strict-assembly --evm-version paris contractSelfDestruct.yul --bin + // Full bytecode: 6002600d60003960026000f3fe30ff + // That's 15 bytes total, padded to 32 bytes with 17 zero bytes at front + mstore(0, 0x0000000000000000000000000000000000000000006002600d60003960026000f3fe30ff) + + // CREATE contract with 100 wei, using deploy bytecode + // The bytecode is 15 bytes, starts at position 17 in the 32-byte word + let contractAddr := create(100, 17, 15) + + // Call the created contract (triggers selfdestruct to self) + pop(call(gas(), contractAddr, 0, 0, 0, 0, 0)) + + // Check contract's balance immediately after selfdestruct + // Store in slot 0 to verify it's 0 (proving immediate burn) + sstore(0, balance(contractAddr)) + + // Send 50 wei to the contract (after it selfdestructed) + pop(call(gas(), contractAddr, 50, 0, 0, 0, 0)) + + // Check balance again after sending funds + // Store in slot 1 to verify it's 0 (funds sent to destroyed contract are burnt) + sstore(1, balance(contractAddr)) + + stop() + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json index e213777848..87797622e1 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json @@ -116,7 +116,8 @@ "0xe1c52dc63b719ade82e8bea94cc41a0d5d28e4aaf536adb5e9cccc9ff8c1aeda" ], "data": "0x0000000000000000000000004f5777744b500616697cb655dcb02ee6cd51deb5be96016bb57376da7a6d296e0a405ee1501778227dfa604df0a81cb1ae018598", - "position": "0x0" + "position": "0x0", + "index": "0x0" }, { "address": "0x200edd17f30485a8735878661960cd7a9a95733f", @@ -124,7 +125,8 @@ "0xacbdb084c721332ac59f9b8e392196c9eb0e4932862da8eb9beaf0dad4f550da" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", - "position": "0x0" + "position": "0x0", + "index": "0x1" } ], "value": "0x8ac7230489e80000", @@ -134,4 +136,4 @@ "value": "0x8ac7230489e80000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json index 5078f854d6..afbdc36778 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json @@ -278,6 +278,7 @@ "0x0000000000000000000000006ca7f214ab2ddbb9a8e1a1e2c8550e3164e9dba5" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x0", "position": "0x0" } ], @@ -300,6 +301,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x1", "position": "0x0" } ], @@ -330,6 +332,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x2", "position": "0x0" } ], @@ -352,6 +355,7 @@ "0x000000000000000000000000950ca4a06c78934a148b7a3ff3ea8fc366f77a06" ], "data": "0x0000000000000000000000000000000000000000000000000041f50e27d56848", + "index": "0x3", "position": "0x0" } ], @@ -416,6 +420,7 @@ "0x0000000000000000000000003de712784baf97260455ae25fb74f574ec9c1add" ], "data": "0x000000000000000000000000000000000000000000000000de0b6b3a76400000", + "index": "0x4", "position": "0x0" } ], @@ -430,4 +435,4 @@ "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json index 6406c8d867..0545fad562 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json @@ -378,6 +378,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x0", "position": "0x0" } ], @@ -392,6 +393,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x1", "position": "0x1" } ], @@ -514,6 +516,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x2", "position": "0x0" } ], @@ -528,6 +531,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x3", "position": "0x1" } ], @@ -717,6 +721,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccd", + "index": "0x5", "position": "0x0" } ], @@ -900,6 +905,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccc", + "index": "0x7", "position": "0x0" } ], @@ -919,6 +925,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0x8", "position": "0x1" } ], @@ -942,6 +949,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0x9", "position": "0x1" } ], @@ -968,6 +976,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0xa", "position": "0x0" } ], @@ -982,6 +991,7 @@ "0x07cf7e805770612a8b2ee8e0bcbba8aa908df5f85fbc4f9e2ef384cf75315038" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", + "index": "0x4", "position": "0x6" }, { @@ -990,6 +1000,7 @@ "0x7027eecbd2a688fc1fa281702b311ed7168571514adfd17014a55d828cb43382" ], "data": "0x000000000000000000000000000000000000000000000004563918244f400000", + "index": "0x6", "position": "0x8" } ], @@ -1067,6 +1078,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000063", + "index": "0xb", "position": "0x0" } ], @@ -1195,6 +1207,7 @@ "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000064", + "index": "0xe", "position": "0x0" } ], @@ -1209,6 +1222,7 @@ "0x4b0bc4f25f8d0b92d2e12b686ba96cd75e4e69325e6cf7b1f3119d14eaf2cbdf" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526", + "index": "0xc", "position": "0x6" }, { @@ -1217,6 +1231,7 @@ "0xf340c079d598119636d42046c6a2d2faf7a68c04aecee516f0e0b8a9e79b8666" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e9652600000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000000", + "index": "0xd", "position": "0x9" } ], @@ -1267,6 +1282,7 @@ "0x0000000000000000000000007498bb5749c9801f1f7e490baf5f966dbfe4e97b" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0xf", "position": "0x0" } ], @@ -1361,6 +1377,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000001" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x10", "position": "0x2" } ], @@ -1455,6 +1472,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000002" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x11", "position": "0x2" } ], @@ -1549,6 +1567,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000003" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x12", "position": "0x2" } ], @@ -1643,6 +1662,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000004" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x13", "position": "0x2" } ], @@ -1737,6 +1757,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000005" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x14", "position": "0x2" } ], @@ -1831,6 +1852,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000006" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x15", "position": "0x2" } ], @@ -1925,6 +1947,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000007" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x16", "position": "0x2" } ], @@ -2019,6 +2042,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000008" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x17", "position": "0x2" } ], @@ -2113,6 +2137,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000009" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x18", "position": "0x2" } ], @@ -2207,6 +2232,7 @@ "0x000000000000000000000000000000000000000000000000000000000000000a" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x19", "position": "0x2" } ], @@ -2260,6 +2286,7 @@ "0x0000000000000000000000007ccbc69292c7a6d7b538c91f3b283de97906cf30" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1a", "position": "0x0" } ], @@ -2282,6 +2309,7 @@ "0x0000000000000000000000001b9ec8ba24630b75a7a958153ffff56dd6d4b6a2" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1c", "position": "0x0" } ], @@ -2304,6 +2332,7 @@ "0x000000000000000000000000c3a2c744ad1f5253c736875b93bacce5b01b060b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1e", "position": "0x0" } ], @@ -2318,6 +2347,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1b", "position": "0x2" }, { @@ -2326,6 +2356,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1d", "position": "0x3" }, { @@ -2334,6 +2365,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1f", "position": "0x4" } ], @@ -2344,4 +2376,4 @@ "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json index 0ca4c8353f..ad3ce19b6f 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json @@ -199,6 +199,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x0", "position": "0x0" }, { @@ -207,6 +208,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1", "position": "0x0" }, { @@ -215,6 +217,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffebebeb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2", "position": "0x0" }, { @@ -223,6 +226,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8888880000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x3", "position": "0x0" }, { @@ -231,6 +235,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb3b3b30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x4", "position": "0x0" }, { @@ -239,6 +244,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x5", "position": "0x0" }, { @@ -247,6 +253,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x6", "position": "0x0" }, { @@ -255,6 +262,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe3e3e30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x7", "position": "0x0" }, { @@ -263,6 +271,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3e3e3e0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x8", "position": "0x0" }, { @@ -271,6 +280,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x9", "position": "0x0" }, { @@ -279,6 +289,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xa", "position": "0x0" }, { @@ -287,6 +298,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xb", "position": "0x0" }, { @@ -295,6 +307,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdbdbdb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xc", "position": "0x0" }, { @@ -303,6 +316,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xd", "position": "0x0" }, { @@ -311,6 +325,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f4f40000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xe", "position": "0x0" }, { @@ -319,6 +334,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xf", "position": "0x0" }, { @@ -327,6 +343,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x10", "position": "0x0" }, { @@ -335,6 +352,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x11", "position": "0x0" }, { @@ -343,6 +361,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x12", "position": "0x0" }, { @@ -351,6 +370,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x13", "position": "0x0" }, { @@ -359,6 +379,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x14", "position": "0x0" }, { @@ -367,6 +388,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb0b0b00000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x15", "position": "0x0" }, { @@ -375,7 +397,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x16", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -383,7 +406,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa0a0a00000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x17", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -391,7 +415,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5b5b5b0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x18", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -399,7 +424,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x19", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -407,7 +433,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1a", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -415,7 +442,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1b", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -423,7 +451,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa9a9a90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1c", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -431,7 +460,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb9b9b90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1d", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -439,7 +469,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1e", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -447,7 +478,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x1f", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -455,7 +487,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x20", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -463,7 +496,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x21", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -471,7 +505,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6363630000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x22", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -479,7 +514,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x23", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -487,7 +523,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9f9f90000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x24", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -495,7 +532,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x25", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -503,7 +541,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9c9c9c0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x26", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -511,7 +550,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8f8f80000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x27", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -519,7 +559,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x28", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -527,7 +568,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x29", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -535,7 +577,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2a", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -543,7 +586,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2b", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -551,7 +595,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2c", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -559,7 +604,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2d", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -567,7 +613,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2e", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -575,7 +622,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4d4e530000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x2f", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -583,7 +631,8 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034300000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x30", + "position": "0x0" }, { "address": "0x350e0ffc780a6a75b44cc52e1ff9092870668945", @@ -591,10 +640,11 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f494b0000000000000000000000000000000000000000000000000011c37937e08000", - "position": "0x0" + "index": "0x31", + "position": "0x0" } ], "value": "0x3782dace9d90000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json index de8ee41af4..58f2c4b8c3 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json @@ -287,6 +287,7 @@ "0xaf30e4d66b2f1f23e63ef4591058a897f67e6867233e33ca3508b982dcc4129b" ], "data": "0x00000000000000000000000050739060a2c32dc076e507ae1a893aab28ecfe68d1b13c1538a940417bf0e73b2498634436753c854c7fb971224d971bd2ae3e8800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000016000000000000000000000000000000000000000000000000000000000000001e000000000000000000000000000000000000000000000000000000000000249f011000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000355524c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000436a736f6e2868747470733a2f2f6170692e72616e646f6d2e6f72672f6a736f6e2d7270632f312f696e766f6b65292e726573756c742e72616e646f6d2e646174612e300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000012c4244584a68725670424a35336f3243786c4a526c51745a4a4b5a714c5974354951652b37335944533448744e6a5335486f64624942337476666f773755717579416b303835566b4c6e4c3945704b67777157517a375a4c64477673516c526432734b78496f6c4e673944626e6650737047714c684c62625953566e4e38437776736a7041586353536f33632b34634e774339307946346f4e69626b764433797461706f5a37676f5453796f5559546677536a6e773374692b484a5648374e332b633069774f43715a6a4464734751556358336d33532f494857624f4f5151356f734f344c626a33476730783155644e7466557a5943465937396e7a596757495145464375524249306e364e42764251573732372b4f73445259304a2f392f676a74387563696248576963303d0000000000000000000000000000000000000000", + "index": "0x1", "position": "0x4" } ], @@ -299,10 +300,11 @@ "address": "0x50739060a2c32dc076e507ae1a893aab28ecfe68", "topics": [], "data": "0x62616e6b726f6c6c5f6d69736d61746368", + "index": "0x0", "position": "0x2" } ], "value": "0x429d069189e0000", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json index 8a68061191..5e79fccabd 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json @@ -96,10 +96,11 @@ "0x000000000000000000000000dbf03b407c01e7cd3cbea99509d93f8dddc8c6fb" ], "data": "0x0000000000000000000000000000000000000000000000000000000000989680", + "index": "0x0", "position": "0x0" } ], "value": "0x0", "type": "CALL" } -} +} \ No newline at end of file diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 7f376a27fc..b21e104abc 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -46,7 +46,7 @@ type vmContext struct { } func testCtx() *vmContext { - return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: uint256.NewInt(100000)}} } func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { @@ -63,7 +63,7 @@ func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCo contract.Code = contractCode } - tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice.ToBig()}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) ret, err := evm.Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) @@ -186,7 +186,7 @@ func TestHaltBetweenSteps(t *testing.T) { Contract: vm.NewContract(common.Address{}, common.Address{}, uint256.NewInt(0), 0, nil), } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(1)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(1)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) @@ -210,7 +210,7 @@ func TestNoStepExec(t *testing.T) { t.Fatal(err) } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(100)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(100)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) tracer.OnExit(0, nil, 0, nil, false) @@ -240,7 +240,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.ByzantiumBlock = big.NewInt(100) chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) - txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} + txCtx := vm.TxContext{GasPrice: uint256.NewInt(100000)} tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil, chaincfg) if err != nil { t.Fatal(err) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 824a5e0c3e..67e07f78d0 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -94,8 +94,8 @@ func (s *StructLog) ErrorString() string { return "" } -// WriteTo writes the human-readable log data into the supplied writer. -func (s *StructLog) WriteTo(writer io.Writer) { +// Write writes the human-readable log data into the supplied writer. +func (s *StructLog) Write(writer io.Writer) { fmt.Fprintf(writer, "%-16spc=%08d gas=%v cost=%v", s.Op, s.Pc, s.Gas, s.GasCost) if s.Err != nil { fmt.Fprintf(writer, " ERROR: %v", s.Err) @@ -324,7 +324,7 @@ func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope l.logs = append(l.logs, entry) return } - log.WriteTo(l.writer) + log.Write(l.writer) } // OnExit is called a call frame finishes processing. @@ -405,7 +405,7 @@ func (l *StructLogger) Output() []byte { return l.output } // @deprecated func WriteTrace(writer io.Writer, logs []StructLog) { for _, log := range logs { - log.WriteTo(writer) + log.Write(writer) } } diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 5ab7ef6ede..e78d27ef36 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -42,6 +42,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` // Position of the log relative to subcalls within the same trace // See https://github.com/ethereum/go-ethereum/pull/28389 for details Position hexutil.Uint `json:"position"` @@ -265,6 +266,7 @@ func (t *callTracer) OnLog(log *types.Log) { Address: log.Address, Topics: log.Topics, Data: log.Data, + Index: hexutil.Uint(log.Index), Position: hexutil.Uint(len(t.callstack[len(t.callstack)-1].Calls)), } t.callstack[len(t.callstack)-1].Logs = append(t.callstack[len(t.callstack)-1].Logs, l) diff --git a/eth/tracers/native/erc7562.go b/eth/tracers/native/erc7562.go index 3ab98c7132..34e202f667 100644 --- a/eth/tracers/native/erc7562.go +++ b/eth/tracers/native/erc7562.go @@ -513,7 +513,7 @@ func defaultIgnoredOpcodes() []hexutil.Uint64 { ignored := make([]hexutil.Uint64, 0, 64) // Allow all PUSHx, DUPx and SWAPx opcodes as they have sequential codes - for op := vm.PUSH0; op < vm.SWAP16; op++ { + for op := vm.PUSH0; op <= vm.SWAP16; op++ { ignored = append(ignored, hexutil.Uint64(op)) } diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 7100a2ddc9..87d18af375 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -160,6 +160,14 @@ func (t *muxTracer) OnCodeChange(a common.Address, prevCodeHash common.Hash, pre } } +func (t *muxTracer) OnCodeChangeV2(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + for _, t := range t.tracers { + if t.OnCodeChangeV2 != nil { + t.OnCodeChangeV2(a, prevCodeHash, prev, codeHash, code, reason) + } + } +} + func (t *muxTracer) OnStorageChange(a common.Address, k, prev, new common.Hash) { for _, t := range t.tracers { if t.OnStorageChange != nil { diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index abd2e4e7ce..6ff5b4f7b4 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -66,7 +66,7 @@ type prestateTracer struct { pre stateMap post stateMap to common.Address - config prestateTracerConfig + config PrestateTracerConfig chainConfig *params.ChainConfig interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption @@ -74,7 +74,7 @@ type prestateTracer struct { deleted map[common.Address]bool } -type prestateTracerConfig struct { +type PrestateTracerConfig struct { DiffMode bool `json:"diffMode"` // If true, this tracer will return state modifications DisableCode bool `json:"disableCode"` // If true, this tracer will not return the contract code DisableStorage bool `json:"disableStorage"` // If true, this tracer will not return the contract storage @@ -82,7 +82,7 @@ type prestateTracerConfig struct { } func newPrestateTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { - var config prestateTracerConfig + var config PrestateTracerConfig if err := json.Unmarshal(cfg, &config); err != nil { return nil, err } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 2c2fbf4550..1a6640f7f7 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -124,7 +124,7 @@ func (ec *Client) PeerCount(ctx context.Context) (uint64, error) { // BlockReceipts returns the receipts of a given block number or hash. func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.Receipt, error) { var r []*types.Receipt - err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash.String()) + err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash) if err == nil && r == nil { return nil, ethereum.NotFound } @@ -518,9 +518,12 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer } func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { - arg := map[string]interface{}{ - "address": q.Addresses, - "topics": q.Topics, + arg := map[string]interface{}{} + if q.Addresses != nil { + arg["address"] = q.Addresses + } + if q.Topics != nil { + arg["topics"] = q.Topics } if q.BlockHash != nil { arg["blockHash"] = *q.BlockHash @@ -747,9 +750,11 @@ func (ec *Client) SendRawTransactionSync( rawTx []byte, timeout *time.Duration, ) (*types.Receipt, error) { - var ms *hexutil.Uint64 + var ms *uint64 if timeout != nil { - if d := hexutil.Uint64(timeout.Milliseconds()); d > 0 { + msInt := timeout.Milliseconds() + if msInt > 0 { + d := uint64(msInt) ms = &d } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 302ccf2e16..f9e761e412 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -687,6 +687,49 @@ func testTransactionSender(t *testing.T, client *rpc.Client) { } } +func TestBlockReceiptsPreservesCanonicalFlag(t *testing.T) { + srv := rpc.NewServer() + service := &blockReceiptsTestService{calls: make(chan rpc.BlockNumberOrHash, 1)} + if err := srv.RegisterName("eth", service); err != nil { + t.Fatalf("failed to register service: %v", err) + } + defer srv.Stop() + + client := rpc.DialInProc(srv) + defer client.Close() + + ec := ethclient.NewClient(client) + defer ec.Close() + + hash := common.HexToHash("0x01") + ref := rpc.BlockNumberOrHashWithHash(hash, true) + + if _, err := ec.BlockReceipts(context.Background(), ref); err != nil { + t.Fatalf("BlockReceipts returned error: %v", err) + } + + select { + case call := <-service.calls: + if call.BlockHash == nil || *call.BlockHash != hash { + t.Fatalf("unexpected block hash: got %v, want %v", call.BlockHash, hash) + } + if !call.RequireCanonical { + t.Fatalf("requireCanonical flag was lost: %+v", call) + } + default: + t.Fatal("service was not called") + } +} + +type blockReceiptsTestService struct { + calls chan rpc.BlockNumberOrHash +} + +func (s *blockReceiptsTestService) GetBlockReceipts(ctx context.Context, block rpc.BlockNumberOrHash) ([]*types.Receipt, error) { + s.calls <- block + return []*types.Receipt{}, nil +} + func newCanceledContext() context.Context { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/ethclient/gethclient/gen_callframe_json.go b/ethclient/gethclient/gen_callframe_json.go new file mode 100644 index 0000000000..48df2790cc --- /dev/null +++ b/ethclient/gethclient/gen_callframe_json.go @@ -0,0 +1,104 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallFrame) MarshalJSON() ([]byte, error) { + type CallFrame0 struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input hexutil.Bytes `json:"input"` + Output hexutil.Bytes `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var enc CallFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.RevertReason = c.RevertReason + enc.Calls = c.Calls + enc.Logs = c.Logs + enc.Value = (*hexutil.Big)(c.Value) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallFrame) UnmarshalJSON(input []byte) error { + type CallFrame0 struct { + Type *string `json:"type"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input *hexutil.Bytes `json:"input"` + Output *hexutil.Bytes `json:"output,omitempty"` + Error *string `json:"error,omitempty"` + RevertReason *string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var dec CallFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.RevertReason != nil { + c.RevertReason = *dec.RevertReason + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Logs != nil { + c.Logs = dec.Logs + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/ethclient/gethclient/gen_calllog_json.go b/ethclient/gethclient/gen_calllog_json.go new file mode 100644 index 0000000000..50e25d4bb3 --- /dev/null +++ b/ethclient/gethclient/gen_calllog_json.go @@ -0,0 +1,61 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callLogMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallLog) MarshalJSON() ([]byte, error) { + type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` + Position hexutil.Uint `json:"position"` + } + var enc CallLog + enc.Address = c.Address + enc.Topics = c.Topics + enc.Data = c.Data + enc.Index = hexutil.Uint(c.Index) + enc.Position = hexutil.Uint(c.Position) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallLog) UnmarshalJSON(input []byte) error { + type CallLog struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data *hexutil.Bytes `json:"data"` + Index *hexutil.Uint `json:"index"` + Position *hexutil.Uint `json:"position"` + } + var dec CallLog + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address != nil { + c.Address = *dec.Address + } + if dec.Topics != nil { + c.Topics = dec.Topics + } + if dec.Data != nil { + c.Data = *dec.Data + } + if dec.Index != nil { + c.Index = uint(*dec.Index) + } + if dec.Position != nil { + c.Position = uint(*dec.Position) + } + return nil +} diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 6a0f5eb312..e677e2bb21 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,10 +19,12 @@ package gethclient import ( "context" + "encoding/json" "fmt" "math/big" "runtime" "runtime/debug" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -104,7 +106,10 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s var res accountResult err := ec.c.CallContext(ctx, &res, "eth_getProof", account, keys, toBlockNumArg(blockNumber)) - // Turn hexutils back to normal datatypes + if err != nil { + return nil, err + } + // Turn hexutils back to normal data types storageResults := make([]StorageResult, 0, len(res.StorageProof)) for _, st := range res.StorageProof { storageResults = append(storageResults, StorageResult{ @@ -122,7 +127,7 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s StorageHash: res.StorageHash, StorageProof: storageResults, } - return &result, err + return &result, nil } // CallContract executes a message call transaction, which is directly executed in the VM @@ -226,6 +231,124 @@ func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *trac return result, nil } +// CallTracerConfig configures the call tracer for +// TraceTransactionWithCallTracer and TraceCallWithCallTracer. +type CallTracerConfig struct { + // OnlyTopCall, when true, limits tracing to the main (top-level) call only. + OnlyTopCall bool + // WithLog, when true, includes log emissions in the trace output. + WithLog bool + // Timeout is the maximum duration the tracer may run. + // Zero means the server default (5s). + Timeout time.Duration +} + +//go:generate go run github.com/fjl/gencodec -type CallLog -field-override callLogMarshaling -out gen_calllog_json.go + +// CallLog represents a log emitted during a traced call. +type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data []byte `json:"data"` + Index uint `json:"index"` + Position uint `json:"position"` +} + +type callLogMarshaling struct { + Data hexutil.Bytes + Index hexutil.Uint + Position hexutil.Uint +} + +//go:generate go run github.com/fjl/gencodec -type CallFrame -field-override callFrameMarshaling -out gen_callframe_json.go + +// CallFrame contains the result of a call tracer run. +type CallFrame struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input []byte `json:"input"` + Output []byte `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *big.Int `json:"value,omitempty"` +} + +type callFrameMarshaling struct { + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Input hexutil.Bytes + Output hexutil.Bytes + Value *hexutil.Big +} + +// TraceTransactionWithCallTracer traces a transaction with the call tracer +// and returns a typed CallFrame. If config is nil, defaults are used. +func (ec *Client) TraceTransactionWithCallTracer(ctx context.Context, txHash common.Hash, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceTransaction", txHash, callTracerConfig(config)) + if err != nil { + return nil, err + } + return &result, nil +} + +// TraceCallWithCallTracer executes a call with the call tracer and returns +// a typed CallFrame. blockNrOrHash selects the block context for the call. +// overrides specifies state overrides (nil for none), blockOverrides specifies +// block header overrides (nil for none), and config configures the tracer +// (nil for defaults). +func (ec *Client) TraceCallWithCallTracer(ctx context.Context, msg ethereum.CallMsg, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceCall", toCallArg(msg), blockNrOrHash, callTraceCallConfig(config, overrides, blockOverrides)) + if err != nil { + return nil, err + } + return &result, nil +} + +// callTracerConfig converts a CallTracerConfig to the wire-format TraceConfig. +func callTracerConfig(config *CallTracerConfig) *tracers.TraceConfig { + tracer := "callTracer" + tc := &tracers.TraceConfig{Tracer: &tracer} + if config != nil { + if config.OnlyTopCall || config.WithLog { + cfg, _ := json.Marshal(struct { + OnlyTopCall bool `json:"onlyTopCall"` + WithLog bool `json:"withLog"` + }{config.OnlyTopCall, config.WithLog}) + tc.TracerConfig = cfg + } + if config.Timeout != 0 { + s := config.Timeout.String() + tc.Timeout = &s + } + } + return tc +} + +// callTraceCallConfig builds the wire-format TraceCallConfig for debug_traceCall, +// bundling tracer settings with optional state and block overrides. +func callTraceCallConfig(config *CallTracerConfig, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides) interface{} { + tc := callTracerConfig(config) + // debug_traceCall expects a single config object that includes both + // tracer settings and any state/block overrides. + type traceCallConfig struct { + *tracers.TraceConfig + StateOverrides map[common.Address]OverrideAccount `json:"stateOverrides,omitempty"` + BlockOverrides *BlockOverrides `json:"blockOverrides,omitempty"` + } + return &traceCallConfig{ + TraceConfig: tc, + StateOverrides: overrides, + BlockOverrides: blockOverrides, + } +} + func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 0eed63cacf..4d8ccfcb6f 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/tracers" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -161,6 +162,12 @@ func TestGethClient(t *testing.T) { }, { "TestCallContractWithBlockOverrides", func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, + }, { + "TestTraceTransactionWithCallTracer", + func(t *testing.T) { testTraceTransactionWithCallTracer(t, client, txHashes) }, + }, { + "TestTraceCallWithCallTracer", + func(t *testing.T) { testTraceCallWithCallTracer(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports // one block. The `testAccessList` fails if the miner has not yet created a @@ -620,3 +627,60 @@ func testCallContractWithBlockOverrides(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected result: %x", res) } } + +func testTraceTransactionWithCallTracer(t *testing.T, client *rpc.Client, txHashes []common.Hash) { + ec := New(client) + for _, txHash := range txHashes { + // With nil config (defaults). + result, err := ec.TraceTransactionWithCallTracer(context.Background(), txHash, nil) + if err != nil { + t.Fatalf("nil config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } + + // With explicit config. + result, err = ec.TraceTransactionWithCallTracer(context.Background(), txHash, + &CallTracerConfig{}, + ) + if err != nil { + t.Fatalf("explicit config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + } +} + +func testTraceCallWithCallTracer(t *testing.T, client *rpc.Client) { + ec := New(client) + msg := ethereum.CallMsg{ + From: testAddr, + To: &common.Address{}, + Gas: 21000, + GasPrice: big.NewInt(1000000000), + Value: big.NewInt(1), + } + result, err := ec.TraceCallWithCallTracer(context.Background(), msg, + rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), nil, nil, nil, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } +} diff --git a/ethclient/types_test.go b/ethclient/types_test.go index 02f9f21758..dcb9a579b7 100644 --- a/ethclient/types_test.go +++ b/ethclient/types_test.go @@ -41,6 +41,18 @@ func TestToFilterArg(t *testing.T) { output interface{} err error }{ + { + "without addresses", + ethereum.FilterQuery{ + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + }, + map[string]interface{}{ + "fromBlock": "0x1", + "toBlock": "0x2", + }, + nil, + }, { "without BlockHash", ethereum.FilterQuery{ diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 7502a6877a..ebb5c9c3e4 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -229,6 +229,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e {TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 32 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 64 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + + // Pebble doesn't use the Bloom filter at level6 for read efficiency. {TargetFileSize: 128 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, } } else { @@ -269,8 +271,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e // limit unchanged allows writes to be flushed more smoothly. This helps // avoid compaction spikes and mitigates write stalls caused by heavy // compaction workloads. - memTableLimit := extraOptions.MemTableStopWritesThreshold - memTableSize := cache * 1024 * 1024 / 2 / memTableLimit + memTableNumber := extraOptions.MemTableStopWritesThreshold + memTableSize := cache * 1024 * 1024 / 2 / memTableNumber // The memory table size is currently capped at maxMemTableSize-1 due to a // known bug in the pebble where maxMemTableSize is not recognized as a @@ -299,12 +301,16 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e // Note, there may have more than two memory tables in the system. MemTableSize: uint64(memTableSize), - // MemTableStopWritesThreshold places a hard limit on the size + // MemTableStopWritesThreshold places a hard limit on the number // of the existent MemTables(including the frozen one). + // // Note, this must be the number of tables not the size of all memtables // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. - MemTableStopWritesThreshold: memTableLimit, + // + // MemTableStopWritesThreshold is set to twice the maximum number of + // allowed memtables to accommodate temporary spikes. + MemTableStopWritesThreshold: memTableNumber * 2, // The default compaction concurrency(1 thread), // Here use all available CPUs for faster compaction. diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index b6191baa12..c17e225165 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -63,6 +63,7 @@ const ( type backend interface { SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription + SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription CurrentHeader() *types.Header HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) Stats() (pending int, queued int) @@ -92,8 +93,9 @@ type Service struct { pongCh chan struct{} // Pong notifications are fed into this channel histCh chan []uint64 // History request block numbers are fed into this channel - headSub event.Subscription - txSub event.Subscription + headSub event.Subscription + txSub event.Subscription + newPayloadSub event.Subscription } // connWrapper is a wrapper to prevent concurrent-write or concurrent-read on the @@ -198,7 +200,9 @@ func (s *Service) Start() error { s.headSub = s.backend.SubscribeChainHeadEvent(chainHeadCh) txEventCh := make(chan core.NewTxsEvent, txChanSize) s.txSub = s.backend.SubscribeNewTxsEvent(txEventCh) - go s.loop(chainHeadCh, txEventCh) + newPayloadCh := make(chan core.NewPayloadEvent, chainHeadChanSize) + s.newPayloadSub = s.backend.SubscribeNewPayloadEvent(newPayloadCh) + go s.loop(chainHeadCh, txEventCh, newPayloadCh) log.Info("Stats daemon started") return nil @@ -208,18 +212,20 @@ func (s *Service) Start() error { func (s *Service) Stop() error { s.headSub.Unsubscribe() s.txSub.Unsubscribe() + s.newPayloadSub.Unsubscribe() log.Info("Stats daemon stopped") return nil } // loop keeps trying to connect to the netstats server, reporting chain events // until termination. -func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent) { +func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent, newPayloadCh chan core.NewPayloadEvent) { // Start a goroutine that exhausts the subscriptions to avoid events piling up var ( - quitCh = make(chan struct{}) - headCh = make(chan *types.Header, 1) - txCh = make(chan struct{}, 1) + quitCh = make(chan struct{}) + headCh = make(chan *types.Header, 1) + txCh = make(chan struct{}, 1) + newPayloadEvCh = make(chan core.NewPayloadEvent, 1) ) go func() { var lastTx mclock.AbsTime @@ -246,11 +252,20 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core default: } + // Notify of new payload events, but drop if too frequent + case ev := <-newPayloadCh: + select { + case newPayloadEvCh <- ev: + default: + } + // node stopped case <-s.txSub.Err(): break HandleLoop case <-s.headSub.Err(): break HandleLoop + case <-s.newPayloadSub.Err(): + break HandleLoop } } close(quitCh) @@ -336,6 +351,10 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core if err = s.reportPending(conn); err != nil { log.Warn("Post-block transaction stats report failed", "err", err) } + case ev := <-newPayloadEvCh: + if err = s.reportNewPayload(conn, ev); err != nil { + log.Warn("New payload stats report failed", "err", err) + } case <-txCh: if err = s.reportPending(conn); err != nil { log.Warn("Transaction stats report failed", "err", err) @@ -600,6 +619,33 @@ func (s uncleStats) MarshalJSON() ([]byte, error) { return []byte("[]"), nil } +// newPayloadStats is the information to report about new payload events. +type newPayloadStats struct { + Number uint64 `json:"number"` + Hash common.Hash `json:"hash"` + ProcessingTime uint64 `json:"processingTime"` // nanoseconds +} + +// reportNewPayload reports a new payload event to the stats server. +func (s *Service) reportNewPayload(conn *connWrapper, ev core.NewPayloadEvent) error { + details := &newPayloadStats{ + Number: ev.Number, + Hash: ev.Hash, + ProcessingTime: uint64(ev.ProcessingTime.Nanoseconds()), + } + + log.Trace("Sending new payload to ethstats", "number", details.Number, "hash", details.Hash) + + stats := map[string]interface{}{ + "id": s.node, + "block": details, + } + report := map[string][]interface{}{ + "emit": {"block_new_payload", stats}, + } + return conn.WriteJSON(report) +} + // reportBlock retrieves the current chain head and reports it to the stats server. func (s *Service) reportBlock(conn *connWrapper, header *types.Header) error { // Gather the block details from the header or block chain diff --git a/go.mod b/go.mod index b4dc9a281a..99f8f4c1ce 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/cockroachdb/pebble v1.1.5 github.com/consensys/gnark-crypto v0.18.0 github.com/crate-crypto/go-eth-kzg v1.4.0 - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a github.com/davecgh/go-spew v1.1.1 github.com/dchest/siphash v1.2.3 github.com/deckarep/golang-set/v2 v2.6.0 @@ -25,7 +24,6 @@ require ( github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 github.com/ethereum/c-kzg-4844/v2 v2.1.5 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab - github.com/ethereum/go-verkle v0.2.2 github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.4 github.com/fsnotify/fsnotify v1.6.0 @@ -35,7 +33,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/snappy v1.0.0 github.com/google/gofuzz v1.2.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v1.3.0 github.com/hashicorp/go-bexpr v0.1.10 @@ -48,11 +46,11 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 + github.com/klauspost/compress v1.17.8 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 - github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/pion/stun/v2 v2.0.0 github.com/pkg/errors v0.9.1 @@ -63,24 +61,42 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/spf13/pflag v1.0.6 github.com/status-im/keycard-go v0.2.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.27.5 + go.opentelemetry.io/otel v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 + go.opentelemetry.io/otel/sdk v1.39.0 + go.opentelemetry.io/otel/trace v1.39.0 go.uber.org/automaxprocs v1.5.2 go.uber.org/goleak v1.3.0 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.36.0 - golang.org/x/text v0.23.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.39.0 + golang.org/x/text v0.31.0 golang.org/x/time v0.9.0 - golang.org/x/tools v0.29.0 - google.golang.org/protobuf v1.34.2 + golang.org/x/tools v0.38.0 + google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.77.0 // indirect +) + require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect @@ -118,10 +134,11 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/grafana/pyroscope-go v1.2.7 + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -142,13 +159,13 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 4ec857c99b..ee578c8ddb 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3M github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/ccoveille/go-safecast v1.1.0 h1:iHKNWaZm+OznO7Eh6EljXPjGfGQsSfa6/sxPlIEKO+g= github.com/ccoveille/go-safecast v1.1.0/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -83,8 +85,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -119,8 +119,6 @@ github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3 github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= @@ -144,6 +142,11 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -178,21 +181,27 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= @@ -226,8 +235,8 @@ github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4 github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -261,7 +270,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -278,8 +286,6 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcou github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= @@ -333,8 +339,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -348,6 +354,8 @@ github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -356,8 +364,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -376,6 +384,24 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -388,16 +414,16 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -413,8 +439,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -423,8 +449,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -457,8 +483,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -476,8 +502,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= @@ -489,21 +515,29 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/graphql/graphql.go b/graphql/graphql.go index aff9205ad9..58b462842b 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -279,7 +279,7 @@ func (t *Transaction) GasPrice(ctx context.Context) hexutil.Big { return hexutil.Big{} } switch tx.Type() { - case types.DynamicFeeTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: if block != nil { if baseFee, _ := block.BaseFeePerGas(ctx); baseFee != nil { // price = min(gasTipCap + baseFee, gasFeeCap) @@ -588,6 +588,9 @@ func (t *Transaction) getLogs(ctx context.Context, hash common.Hash) (*[]*Log, e func (t *Transaction) Type(ctx context.Context) *hexutil.Uint64 { tx, _ := t.resolve(ctx) + if tx == nil { + return nil + } txType := hexutil.Uint64(tx.Type()) return &txType } @@ -717,6 +720,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { if err != nil { return nil, err } + if b.header == nil { + return nil, nil + } if b.hash == (common.Hash{}) { b.hash = b.header.Hash() } @@ -1441,7 +1447,7 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics, 0) return runFilter(ctx, r, filter) } diff --git a/internal/build/file.go b/internal/build/file.go index 2cd090c42c..b7c00eb842 100644 --- a/internal/build/file.go +++ b/internal/build/file.go @@ -21,20 +21,21 @@ import ( "io" "os" "path/filepath" - "sort" - "strings" + "slices" ) // HashFolder iterates all files under the given directory, computing the hash // of each. -func HashFolder(folder string, exlude []string) (map[string][32]byte, error) { +func HashFolder(folder string, excludes []string) (map[string][32]byte, error) { res := make(map[string][32]byte) err := filepath.WalkDir(folder, func(path string, d os.DirEntry, _ error) error { // Skip anything that's exluded or not a regular file - for _, skip := range exlude { - if strings.HasPrefix(path, filepath.FromSlash(skip)) { + // Skip anything that's excluded or not a regular file + if slices.Contains(excludes, path) { + if d.IsDir() { return filepath.SkipDir } + return nil } if !d.Type().IsRegular() { return nil @@ -71,6 +72,6 @@ func DiffHashes(a map[string][32]byte, b map[string][32]byte) []string { updates = append(updates, file) } } - sort.Strings(updates) + slices.Sort(updates) return updates } diff --git a/internal/debug/api.go b/internal/debug/api.go index 1bac36e908..5a2781cc77 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -252,7 +252,7 @@ func (*HandlerT) SetGCPercent(v int) int { // - Geth also allocates memory off-heap, particularly for fastCache and Pebble, // which can be non-trivial (a few gigabytes by default). func (*HandlerT) SetMemoryLimit(limit int64) int64 { - log.Info("Setting memory limit", "size", common.PrettyDuration(limit)) + log.Info("Setting memory limit", "size", common.StorageSize(limit)) return debug.SetMemoryLimit(limit) } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 3b865bd1df..0639f57bb0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -162,6 +162,11 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, + pyroscopeFlag, + pyroscopeServerFlag, + pyroscopeAuthUsernameFlag, + pyroscopeAuthPasswordFlag, + pyroscopeTagsFlag, } var ( @@ -300,6 +305,14 @@ func Setup(ctx *cli.Context) error { // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) } + + // Pyroscope profiling + if ctx.Bool(pyroscopeFlag.Name) { + if err := startPyroscope(ctx); err != nil { + return err + } + } + if len(logFile) > 0 || rotation { log.Info("Logging configured", context...) } @@ -323,6 +336,7 @@ func StartPProf(address string, withMetrics bool) { // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { + stopPyroscope() Handler.StopCPUProfile() Handler.StopGoTrace() if logOutputFile != nil { diff --git a/internal/debug/pyroscope.go b/internal/debug/pyroscope.go new file mode 100644 index 0000000000..d0804cb891 --- /dev/null +++ b/internal/debug/pyroscope.go @@ -0,0 +1,134 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package debug + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/grafana/pyroscope-go" + "github.com/urfave/cli/v2" +) + +var ( + pyroscopeFlag = &cli.BoolFlag{ + Name: "pyroscope", + Usage: "Enable Pyroscope profiling", + Value: false, + Category: flags.LoggingCategory, + } + pyroscopeServerFlag = &cli.StringFlag{ + Name: "pyroscope.server", + Usage: "Pyroscope server URL to push profiling data to", + Value: "http://localhost:4040", + Category: flags.LoggingCategory, + } + pyroscopeAuthUsernameFlag = &cli.StringFlag{ + Name: "pyroscope.username", + Usage: "Pyroscope basic authentication username", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeAuthPasswordFlag = &cli.StringFlag{ + Name: "pyroscope.password", + Usage: "Pyroscope basic authentication password", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeTagsFlag = &cli.StringFlag{ + Name: "pyroscope.tags", + Usage: "Comma separated list of key=value tags to add to profiling data", + Value: "", + Category: flags.LoggingCategory, + } +) + +// This holds the globally-configured Pyroscope instance. +var pyroscopeProfiler *pyroscope.Profiler + +func startPyroscope(ctx *cli.Context) error { + server := ctx.String(pyroscopeServerFlag.Name) + authUsername := ctx.String(pyroscopeAuthUsernameFlag.Name) + authPassword := ctx.String(pyroscopeAuthPasswordFlag.Name) + + rawTags := ctx.String(pyroscopeTagsFlag.Name) + tags := make(map[string]string) + for tag := range strings.SplitSeq(rawTags, ",") { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } + k, v, _ := strings.Cut(tag, "=") + tags[k] = v + } + + config := pyroscope.Config{ + ApplicationName: "geth", + ServerAddress: server, + BasicAuthUser: authUsername, + BasicAuthPassword: authPassword, + Logger: &pyroscopeLogger{Logger: log.Root()}, + Tags: tags, + ProfileTypes: []pyroscope.ProfileType{ + // Enabling all profile types + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + } + + profiler, err := pyroscope.Start(config) + if err != nil { + return err + } + pyroscopeProfiler = profiler + log.Info("Enabled Pyroscope") + return nil +} + +func stopPyroscope() { + if pyroscopeProfiler != nil { + pyroscopeProfiler.Stop() + pyroscopeProfiler = nil + } +} + +// Small wrapper for log.Logger to satisfy pyroscope.Logger interface +type pyroscopeLogger struct { + Logger log.Logger +} + +func (l *pyroscopeLogger) Infof(format string, v ...any) { + l.Logger.Info(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Debugf(format string, v ...any) { + l.Logger.Debug(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Errorf(format string, v ...any) { + l.Logger.Error(fmt.Sprintf("Pyroscope: "+format, v...)) +} diff --git a/internal/download/download.go b/internal/download/download.go index 26c7795ce5..c59c8a90c3 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -205,7 +205,10 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { if err != nil { return err } - dst := newDownloadWriter(fd, resp.ContentLength) + var dst io.WriteCloser = fd + if resp.ContentLength > 0 { + dst = newDownloadWriter(fd, resp.ContentLength) + } _, err = io.Copy(dst, resp.Body) dst.Close() if err != nil { diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go index 83a761f1fd..72e36fe00f 100644 --- a/internal/era/accumulator.go +++ b/internal/era/accumulator.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "slices" "github.com/ethereum/go-ethereum/common" ssz "github.com/ferranbt/fastssz" @@ -31,8 +32,8 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro if len(hashes) != len(tds) { return common.Hash{}, errors.New("must have equal number hashes as td values") } - if len(hashes) > MaxEra1Size { - return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + if len(hashes) > MaxSize { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxSize) } hh := ssz.NewHasher() for i := range hashes { @@ -43,7 +44,7 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro } hh.Append(root[:]) } - hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxSize)) return hh.HashRoot() } @@ -69,23 +70,15 @@ func (h *headerRecord) HashTreeRoot() ([32]byte, error) { // HashTreeRootWith ssz hashes the headerRecord object with a hasher. func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { hh.PutBytes(h.Hash[:]) - td := bigToBytes32(h.TotalDifficulty) + td := BigToBytes32(h.TotalDifficulty) hh.PutBytes(td[:]) hh.Merkleize(0) return } // bigToBytes32 converts a big.Int into a little-endian 32-byte array. -func bigToBytes32(n *big.Int) (b [32]byte) { +func BigToBytes32(n *big.Int) (b [32]byte) { n.FillBytes(b[:]) - reverseOrder(b[:]) + slices.Reverse(b[:]) return } - -// reverseOrder reverses the byte order of a slice. -func reverseOrder(b []byte) []byte { - for i := 0; i < 16; i++ { - b[i], b[32-i-1] = b[32-i-1], b[i] - } - return b -} diff --git a/internal/era/era.go b/internal/era/era.go index 118c67abfd..a3c8465bc4 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -1,4 +1,4 @@ -// Copyright 2024 The go-ethereum Authors +// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,7 +17,6 @@ package era import ( - "encoding/binary" "fmt" "io" "math/big" @@ -25,293 +24,132 @@ import ( "path" "strconv" "strings" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/internal/era/e2store" - "github.com/ethereum/go-ethereum/rlp" - "github.com/golang/snappy" ) +// Type constants for the e2store entries in the Era1 and EraE formats. var ( - TypeVersion uint16 = 0x3265 - TypeCompressedHeader uint16 = 0x03 - TypeCompressedBody uint16 = 0x04 - TypeCompressedReceipts uint16 = 0x05 - TypeTotalDifficulty uint16 = 0x06 - TypeAccumulator uint16 = 0x07 - TypeBlockIndex uint16 = 0x3266 - - MaxEra1Size = 8192 + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeCompressedSlimReceipts uint16 = 0x0a // uses eth/69 encoding + TypeProof uint16 = 0x0b + TypeBlockIndex uint16 = 0x3266 + TypeComponentIndex uint16 = 0x3267 + + MaxSize = 8192 + // headerSize uint64 = 8 ) -// Filename returns a recognizable Era1-formatted file name for the specified -// epoch and network. -func Filename(network string, epoch int, root common.Hash) string { - return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer } -// ReadDir reads all the era1 files in a directory for a given network. -// Format: --.era1 +// Iterator provides sequential access to blocks in an era file. +type Iterator interface { + // Next advances to the next block. Returns true if a block is available, + // false when iteration is complete or an error occurred. + Next() bool + + // Number returns the block number of the current block. + Number() uint64 + + // Block returns the current block. + Block() (*types.Block, error) + + // BlockAndReceipts returns the current block and its receipts. + BlockAndReceipts() (*types.Block, types.Receipts, error) + + // Receipts returns the receipts for the current block. + Receipts() (types.Receipts, error) + + // Error returns any error encountered during iteration. + Error() error +} + +// Builder constructs era files from blocks and receipts. +// +// Builders handle three epoch types automatically: +// - Pre-merge: all blocks have difficulty > 0, TD is stored for each block +// - Transition: starts pre-merge, ends post-merge; TD stored for all blocks +// - Post-merge: all blocks have difficulty == 0, no TD stored +type Builder interface { + // Add appends a block and its receipts to the era file. + // For pre-merge blocks, td must be provided. + // For post-merge blocks, td should be nil. + Add(block *types.Block, receipts types.Receipts, td *big.Int) error + + // AddRLP appends RLP-encoded block components to the era file. + // For pre-merge blocks, td and difficulty must be provided. + // For post-merge blocks, td and difficulty should be nil. + AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error + + // Finalize writes all collected entries and returns the epoch identifier. + // For Era1 (onedb): returns the accumulator root. + // For EraE (execdb): returns the last block hash. + Finalize() (common.Hash, error) + + // Accumulator returns the accumulator root after Finalize has been called. + // Returns nil for post-merge epochs where no accumulator exists. + Accumulator() *common.Hash +} + +// Era represents the interface for reading era data. +type Era interface { + Close() error + Start() uint64 + Count() uint64 + Iterator() (Iterator, error) + GetBlockByNumber(num uint64) (*types.Block, error) + GetRawBodyByNumber(num uint64) ([]byte, error) + GetRawReceiptsByNumber(num uint64) ([]byte, error) + InitialTD() (*big.Int, error) + Accumulator() (common.Hash, error) +} + +// ReadDir reads all the era files in a directory for a given network. +// Format: --.erae or --.era1 func ReadDir(dir, network string) ([]string, error) { entries, err := os.ReadDir(dir) + if err != nil { return nil, fmt.Errorf("error reading directory %s: %w", dir, err) } var ( - next = uint64(0) - eras []string + next = uint64(0) + eras []string + dirType string ) for _, entry := range entries { - if path.Ext(entry.Name()) != ".era1" { + ext := path.Ext(entry.Name()) + if ext != ".erae" && ext != ".era1" { continue } + if dirType == "" { + dirType = ext + } parts := strings.Split(entry.Name(), "-") if len(parts) != 3 || parts[0] != network { - // Invalid era1 filename, skip. + // Invalid era filename, skip. continue } if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + return nil, fmt.Errorf("malformed era filenames: %s", entry.Name()) } else if epoch != next { return nil, fmt.Errorf("missing epoch %d", next) } + if dirType != ext { + return nil, fmt.Errorf("directory %s contains mixed era file formats: want %s, have %s", dir, dirType, ext) + } next += 1 eras = append(eras, entry.Name()) } return eras, nil } - -type ReadAtSeekCloser interface { - io.ReaderAt - io.Seeker - io.Closer -} - -// Era reads and Era1 file. -type Era struct { - f ReadAtSeekCloser // backing era1 file - s *e2store.Reader // e2store reader over f - m metadata // start, count, length info - mu *sync.Mutex // lock for buf - buf [8]byte // buffer reading entry offsets -} - -// From returns an Era backed by f. -func From(f ReadAtSeekCloser) (*Era, error) { - m, err := readMetadata(f) - if err != nil { - return nil, err - } - return &Era{ - f: f, - s: e2store.NewReader(f), - m: m, - mu: new(sync.Mutex), - }, nil -} - -// Open returns an Era backed by the given filename. -func Open(filename string) (*Era, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - return From(f) -} - -func (e *Era) Close() error { - return e.f.Close() -} - -// GetBlockByNumber returns the block for the given block number. -func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) - if err != nil { - return nil, err - } - var header types.Header - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - var body types.Body - if err := rlp.Decode(r, &body); err != nil { - return nil, err - } - return types.NewBlockWithHeader(&header).WithBody(body), nil -} - -// GetRawBodyByNumber returns the RLP-encoded body for the given block number. -func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - off, err = e.s.SkipN(off, 1) - if err != nil { - return nil, err - } - r, _, err := newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. -func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - r, _, err := newSnappyReader(e.s, TypeCompressedReceipts, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// Accumulator reads the accumulator entry in the Era1 file. -func (e *Era) Accumulator() (common.Hash, error) { - entry, err := e.s.Find(TypeAccumulator) - if err != nil { - return common.Hash{}, err - } - return common.BytesToHash(entry.Value), nil -} - -// InitialTD returns initial total difficulty before the difficulty of the -// first block of the Era1 is applied. -func (e *Era) InitialTD() (*big.Int, error) { - var ( - r io.Reader - header types.Header - rawTd []byte - n int64 - off int64 - err error - ) - - // Read first header. - if off, err = e.readOffset(e.m.start); err != nil { - return nil, err - } - if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { - return nil, err - } - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - // Read total difficulty after first block. - if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { - return nil, err - } - rawTd, err = io.ReadAll(r) - if err != nil { - return nil, err - } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) - return td.Sub(td, header.Difficulty), nil -} - -// Start returns the listed start block. -func (e *Era) Start() uint64 { - return e.m.start -} - -// Count returns the total number of blocks in the Era1. -func (e *Era) Count() uint64 { - return e.m.count -} - -// readOffset reads a specific block's offset from the block index. The value n -// is the absolute block number desired. -func (e *Era) readOffset(n uint64) (int64, error) { - var ( - blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header - firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num - indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes - offOffset = firstIndex + indexOffset // offset of block offset - ) - e.mu.Lock() - defer e.mu.Unlock() - clear(e.buf[:]) - if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { - return 0, err - } - // Since the block offset is relative from the start of the block index record - // we need to add the record offset to it's offset to get the block's absolute - // offset. - return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil -} - -// newSnappyReader returns a snappy.Reader for the e2store entry value at off. -func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { - r, n, err := e.ReaderAt(expectedType, off) - if err != nil { - return nil, 0, err - } - return snappy.NewReader(r), int64(n), err -} - -// metadata wraps the metadata in the block index. -type metadata struct { - start uint64 - count uint64 - length int64 -} - -// readMetadata reads the metadata stored in an Era1 file's block index. -func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { - // Determine length of reader. - if m.length, err = f.Seek(0, io.SeekEnd); err != nil { - return - } - b := make([]byte, 16) - // Read count. It's the last 8 bytes of the file. - if _, err = f.ReadAt(b[:8], m.length-8); err != nil { - return - } - m.count = binary.LittleEndian.Uint64(b) - // Read start. It's at the offset -sizeof(m.count) - - // count*sizeof(indexEntry) - sizeof(m.start) - if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { - return - } - m.start = binary.LittleEndian.Uint64(b[8:]) - return -} diff --git a/internal/era/eradl/eradl.go b/internal/era/eradl/eradl.go index 30bd2bc0d5..375b9ad15b 100644 --- a/internal/era/eradl/eradl.go +++ b/internal/era/eradl/eradl.go @@ -86,8 +86,8 @@ func (l *Loader) DownloadAll(destDir string) error { // DownloadBlockRange fetches the era1 files for the given block range. func (l *Loader) DownloadBlockRange(start, end uint64, destDir string) error { - startEpoch := start / uint64(era.MaxEra1Size) - endEpoch := end / uint64(era.MaxEra1Size) + startEpoch := start / uint64(era.MaxSize) + endEpoch := end / uint64(era.MaxSize) return l.DownloadEpochRange(startEpoch, endEpoch, destDir) } diff --git a/internal/era/execdb/builder.go b/internal/era/execdb/builder.go new file mode 100644 index 0000000000..6246b9caae --- /dev/null +++ b/internal/era/execdb/builder.go @@ -0,0 +1,332 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +// EraE file format specification. +// +// The format can be summarized with the following expression: +// +// eraE := Version | CompressedHeader* | CompressedBody* | CompressedSlimReceipts* | TotalDifficulty* | other-entries* | Accumulator? | ComponentIndex +// +// Each basic element is its own e2store entry: +// +// Version = { type: 0x3265, data: nil } +// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } +// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } +// CompressedSlimReceipts = { type: 0x0a, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } +// TotalDifficulty = { type: 0x06, data: uint256 (header.total_difficulty) } +// AccumulatorRoot = { type: 0x07, data: hash_tree_root(List(HeaderRecord, 8192)) } +// ComponentIndex = { type: 0x3267, data: component-index } +// +// Notes: +// - TotalDifficulty is present for pre-merge and merge transition epochs. +// For pure post-merge epochs, TotalDifficulty is omitted entirely. +// - In merge transition epochs, post-merge blocks store the final total +// difficulty (the TD at which the merge occurred). +// - AccumulatorRoot is only written for pre-merge epochs. +// - HeaderRecord is defined in the Portal Network specification. +// - Proofs (type 0x09) are defined in the spec but not yet supported in this implementation. +// +// ComponentIndex stores relative offsets to each block's components: +// +// component-index := starting-number | indexes | indexes | ... | component-count | count +// indexes := header-offset | body-offset | receipts-offset | td-offset? +// +// All values are little-endian uint64. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in an +// EraE file is also 8192. + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to build an EraE e2store file. It collects block entries and +// writes them to the underlying e2store.Writer. +type Builder struct { + w *e2store.Writer + + headers [][]byte + hashes []common.Hash // only pre-merge block hashes, for accumulator + bodies [][]byte + receipts [][]byte + tds []*big.Int + + startNum *uint64 + ttd *big.Int // terminal total difficulty + last common.Hash // hash of last block added + accumulator *common.Hash // accumulator root, set by Finalize (nil for post-merge) + + written uint64 + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) era.Builder { + return &Builder{ + w: e2store.NewWriter(w), + } +} + +// Add writes a block entry and its receipts into the e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return fmt.Errorf("encode header: %w", err) + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return fmt.Errorf("encode body: %w", err) + } + + rs := make([]*types.SlimReceipt, len(receipts)) + for i, receipt := range receipts { + rs[i] = (*types.SlimReceipt)(receipt) + } + er, err := rlp.EncodeToBytes(rs) + if err != nil { + return fmt.Errorf("encode receipts: %w", err) + } + + return b.AddRLP(eh, eb, er, block.Number().Uint64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file. +// The builder automatically handles transition epochs where both pre and post-merge blocks exist. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error { + if len(b.headers) >= era.MaxSize { + return fmt.Errorf("exceeds max size %d", era.MaxSize) + } + // Set starting block number on first add. + if b.startNum == nil { + b.startNum = new(uint64) + *b.startNum = number + } + + if difficulty == nil { + return fmt.Errorf("invalid block: difficulty is nil") + } + hasDifficulty := difficulty.Sign() > 0 + // Expect td to be nil for post-merge blocks + // and non-nil for pre-merge blocks. + if hasDifficulty != (td != nil) { + return fmt.Errorf("TD and difficulty mismatch: expected both nil or both non-nil") + } + // After the merge, difficulty must be nil. + post := (b.tds == nil && len(b.headers) > 0) || b.ttd != nil + if post && hasDifficulty { + return fmt.Errorf("post-merge epoch: cannot accept total difficulty for block %d", number) + } + + // If this marks the start of the transition, record final total + // difficulty value. + if b.ttd == nil && len(b.tds) > 0 && !hasDifficulty { + b.ttd = new(big.Int).Set(b.tds[len(b.tds)-1]) + } + + // Record block data. + b.headers = append(b.headers, header) + b.bodies = append(b.bodies, body) + b.receipts = append(b.receipts, receipts) + b.last = blockHash + + // Conditionally write the total difficulty and block hashes. + // - Pre-merge: store total difficulty and block hashes. + // - Transition: only store total difficulty. + // - Post-merge: store neither. + if hasDifficulty { + b.hashes = append(b.hashes, blockHash) + b.tds = append(b.tds, new(big.Int).Set(td)) + } else if b.ttd != nil { + b.tds = append(b.tds, new(big.Int).Set(b.ttd)) + } else { + // Post-merge: no TD or block hashes stored. + } + + return nil +} + +// Accumulator returns the accumulator root after Finalize has been called. +// Returns nil for post-merge epochs where no accumulator exists. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + +type offsets struct { + headers []uint64 + bodies []uint64 + receipts []uint64 + tds []uint64 +} + +// Finalize writes all collected block entries to the e2store file. +// For pre-merge or transition epochs, the accumulator root is computed over +// pre-merge blocks and written. For pure post-merge epochs, no accumulator +// is written. Always returns the last block hash as the epoch identifier. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, errors.New("no blocks added, cannot finalize") + } + // Write version before writing any blocks. + if n, err := b.w.Write(era.TypeVersion, nil); err != nil { + return common.Hash{}, fmt.Errorf("write version entry: %w", err) + } else { + b.written += uint64(n) + } + + // Convert TD values to byte-level LE representation. + var tds [][]byte + for _, td := range b.tds { + tds = append(tds, uint256LE(td)) + } + + // Create snappy writer. + b.buf = bytes.NewBuffer(nil) + b.snappy = snappy.NewBufferedWriter(b.buf) + + var o offsets + for _, section := range []struct { + typ uint16 + data [][]byte + compressed bool + offsets *[]uint64 + }{ + {era.TypeCompressedHeader, b.headers, true, &o.headers}, + {era.TypeCompressedBody, b.bodies, true, &o.bodies}, + {era.TypeCompressedSlimReceipts, b.receipts, true, &o.receipts}, + {era.TypeTotalDifficulty, tds, false, &o.tds}, + } { + for _, data := range section.data { + *section.offsets = append(*section.offsets, b.written) + if section.compressed { + // Write snappy compressed data. + if err := b.snappyWrite(section.typ, data); err != nil { + return common.Hash{}, err + } + } else { + // Directly write uncompressed data. + n, err := b.w.Write(section.typ, data) + if err != nil { + return common.Hash{}, err + } + b.written += uint64(n) + } + } + } + + // Compute and write accumulator root only for epochs that started pre-merge. + // The accumulator is computed over only the pre-merge blocks (b.hashes). + // Pure post-merge epochs have no accumulator. + if len(b.tds) > 0 { + accRoot, err := era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)]) + if err != nil { + return common.Hash{}, fmt.Errorf("compute accumulator: %w", err) + } + if n, err := b.w.Write(era.TypeAccumulator, accRoot[:]); err != nil { + return common.Hash{}, fmt.Errorf("write accumulator: %w", err) + } else { + b.written += uint64(n) + } + b.accumulator = &accRoot + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil + } + + // Pure post-merge epoch: no accumulator. + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil +} + +// uin256LE writes 32 byte big integers to little endian. +func uint256LE(v *big.Int) []byte { + b := v.FillBytes(make([]byte, 32)) + for i := 0; i < 16; i++ { + b[i], b[31-i] = b[31-i], b[i] + } + return b +} + +// SnappyWrite compresses the input data using snappy and writes it to the e2store file. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + b.buf.Reset() + b.snappy.Reset(b.buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := b.snappy.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += uint64(n) + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} + +// writeIndex writes the component index to the file. +func (b *Builder) writeIndex(o *offsets) error { + count := len(o.headers) + + // Post-merge, we only index headers, bodies, and receipts. Pre-merge, we also + // need to index the total difficulties. + componentCount := 3 + if len(o.tds) > 0 { + componentCount++ + } + + // Offsets are stored relative to the index position (negative, stored as uint64). + base := int64(b.written) + rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) } + + var buf bytes.Buffer + write := func(v uint64) { binary.Write(&buf, binary.LittleEndian, v) } + + write(*b.startNum) + for i := range o.headers { + write(rel(o.headers[i])) + write(rel(o.bodies[i])) + write(rel(o.receipts[i])) + if len(o.tds) > 0 { + write(rel(o.tds[i])) + } + } + write(uint64(componentCount)) + write(uint64(count)) + + n, err := b.w.Write(era.TypeComponentIndex, buf.Bytes()) + b.written += uint64(n) + return err +} diff --git a/internal/era/execdb/era_test.go b/internal/era/execdb/era_test.go new file mode 100644 index 0000000000..f66931b9ed --- /dev/null +++ b/internal/era/execdb/era_test.go @@ -0,0 +1,348 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "bytes" + "fmt" + "io" + "math/big" + "os" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +func TestEraE(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + start uint64 + preMerge int + postMerge int + accumulator bool // whether accumulator should exist + }{ + { + name: "pre-merge", + start: 0, + preMerge: 128, + postMerge: 0, + accumulator: true, + }, + { + name: "post-merge", + start: 0, + preMerge: 0, + postMerge: 64, + accumulator: false, + }, + { + name: "transition", + start: 0, + preMerge: 32, + postMerge: 32, + accumulator: true, + }, + { + name: "non-zero-start", + start: 8192, + preMerge: 64, + postMerge: 0, + accumulator: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + // Build test data. + type blockData struct { + header, body, receipts []byte + hash common.Hash + td *big.Int + difficulty *big.Int + } + var ( + builder = NewBuilder(f) + blocks []blockData + totalBlocks = tt.preMerge + tt.postMerge + finalTD = big.NewInt(int64(tt.preMerge)) + ) + + // Add pre-merge blocks. + for i := 0; i < tt.preMerge; i++ { + num := tt.start + uint64(i) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(1)}), + body: mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}}), + receipts: mustEncode([]types.SlimReceipt{{CumulativeGasUsed: uint64(i)}}), + hash: common.Hash{byte(i)}, + td: big.NewInt(int64(i + 1)), + difficulty: big.NewInt(1), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, blk.td, blk.difficulty); err != nil { + t.Fatalf("error adding pre-merge block %d: %v", i, err) + } + } + + // Add post-merge blocks. + for i := 0; i < tt.postMerge; i++ { + idx := tt.preMerge + i + num := tt.start + uint64(idx) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(0)}), + body: mustEncode(&types.Body{}), + receipts: mustEncode([]types.SlimReceipt{}), + hash: common.Hash{byte(idx)}, + difficulty: big.NewInt(0), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, nil, big.NewInt(0)); err != nil { + t.Fatalf("error adding post-merge block %d: %v", idx, err) + } + } + + // Finalize and check return values. + epochID, err := builder.Finalize() + if err != nil { + t.Fatalf("error finalizing: %v", err) + } + + // Verify epoch ID is always the last block hash. + expectedLastHash := blocks[len(blocks)-1].hash + if epochID != expectedLastHash { + t.Fatalf("wrong epoch ID: want %s, got %s", expectedLastHash.Hex(), epochID.Hex()) + } + + // Verify accumulator presence. + if tt.accumulator { + if builder.Accumulator() == nil { + t.Fatal("expected non-nil accumulator") + } + } else { + if builder.Accumulator() != nil { + t.Fatalf("expected nil accumulator, got %s", builder.Accumulator().Hex()) + } + } + + // Open and verify the era file. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + // Verify metadata. + if e.Start() != tt.start { + t.Fatalf("wrong start block: want %d, got %d", tt.start, e.Start()) + } + if e.Count() != uint64(totalBlocks) { + t.Fatalf("wrong block count: want %d, got %d", totalBlocks, e.Count()) + } + + // Verify accumulator in file. + if tt.accumulator { + accRoot, err := e.Accumulator() + if err != nil { + t.Fatalf("error getting accumulator: %v", err) + } + if accRoot != *builder.Accumulator() { + t.Fatalf("accumulator mismatch: builder has %s, file contains %s", + builder.Accumulator().Hex(), accRoot.Hex()) + } + } else { + if _, err := e.Accumulator(); err == nil { + t.Fatal("expected error when reading accumulator from post-merge epoch") + } + } + + // Verify blocks via raw iterator. + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %v", err) + } + for i := 0; i < totalBlocks; i++ { + if !it.Next() { + t.Fatalf("expected more entries at %d", i) + } + if it.Error() != nil { + t.Fatalf("unexpected error: %v", it.Error()) + } + + // Check header. + rawHeader, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(rawHeader, blocks[i].header) { + t.Fatalf("mismatched header at %d", i) + } + + // Check body. + rawBody, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(rawBody, blocks[i].body) { + t.Fatalf("mismatched body at %d", i) + } + + // Check receipts. + rawReceipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(rawReceipts, blocks[i].receipts) { + t.Fatalf("mismatched receipts at %d", i) + } + + // Check TD (only for epochs that have TD stored). + if tt.preMerge > 0 && it.TotalDifficulty != nil { + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading TD: %v", err) + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + var expectedTD *big.Int + if i < tt.preMerge { + expectedTD = blocks[i].td + } else { + // Post-merge blocks in transition epoch use final TD. + expectedTD = finalTD + } + if td.Cmp(expectedTD) != 0 { + t.Fatalf("mismatched TD at %d: want %s, got %s", i, expectedTD, td) + } + } + } + + // Verify random access. + for _, blockNum := range []uint64{tt.start, tt.start + uint64(totalBlocks) - 1} { + blk, err := e.GetBlockByNumber(blockNum) + if err != nil { + t.Fatalf("error getting block %d: %v", blockNum, err) + } + if blk.Number().Uint64() != blockNum { + t.Fatalf("wrong block number: want %d, got %d", blockNum, blk.Number().Uint64()) + } + } + + // Verify out-of-range access fails. + if _, err := e.GetBlockByNumber(tt.start + uint64(totalBlocks)); err == nil { + t.Fatal("expected error for out-of-range block") + } + if tt.start > 0 { + if _, err := e.GetBlockByNumber(tt.start - 1); err == nil { + t.Fatal("expected error for block before start") + } + } + + // Verify high-level iterator. + hlIt, err := e.Iterator() + if err != nil { + t.Fatalf("failed to create iterator: %v", err) + } + count := 0 + for hlIt.Next() { + blk, err := hlIt.Block() + if err != nil { + t.Fatalf("error getting block: %v", err) + } + if blk.Number().Uint64() != tt.start+uint64(count) { + t.Fatalf("wrong block number: want %d, got %d", tt.start+uint64(count), blk.Number().Uint64()) + } + count++ + } + if hlIt.Error() != nil { + t.Fatalf("iterator error: %v", hlIt.Error()) + } + if count != totalBlocks { + t.Fatalf("wrong iteration count: want %d, got %d", totalBlocks, count) + } + }) + } +} + +// TestInitialTD tests the InitialTD calculation separately since it requires +// specific TD/difficulty values. +func TestInitialTD(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-initial-td-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + builder := NewBuilder(f) + + // First block: difficulty=5, TD=10, so initial TD = 10-5 = 5. + header := mustEncode(&types.Header{Number: big.NewInt(0), Difficulty: big.NewInt(5)}) + body := mustEncode(&types.Body{}) + receipts := mustEncode([]types.SlimReceipt{}) + + if err := builder.AddRLP(header, body, receipts, 0, common.Hash{0}, big.NewInt(10), big.NewInt(5)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + // Second block: difficulty=3, TD=13. + header2 := mustEncode(&types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(3)}) + if err := builder.AddRLP(header2, body, receipts, 1, common.Hash{1}, big.NewInt(13), big.NewInt(3)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing: %v", err) + } + + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + initialTD, err := e.InitialTD() + if err != nil { + t.Fatalf("error getting initial TD: %v", err) + } + + // Initial TD should be TD[0] - Difficulty[0] = 10 - 5 = 5. + if initialTD.Cmp(big.NewInt(5)) != 0 { + t.Fatalf("wrong initial TD: want 5, got %s", initialTD) + } +} + +func mustEncode(obj any) []byte { + b, err := rlp.EncodeToBytes(obj) + if err != nil { + panic(fmt.Sprintf("failed to encode obj: %v", err)) + } + return b +} diff --git a/internal/era/execdb/iterator.go b/internal/era/execdb/iterator.go new file mode 100644 index 0000000000..8d17ac00a9 --- /dev/null +++ b/internal/era/execdb/iterator.go @@ -0,0 +1,240 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "errors" + "io" + "math/big" + "slices" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +type Iterator struct { + inner *RawIterator + block *types.Block // cache for decoded block +} + +// NewIterator returns a header/body/receipt iterator over the archive. +// Call Next immediately to position on the first block. +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) + if err != nil { + return nil, err + } + return &Iterator{inner: inner}, nil +} + +// Next advances to the next block entry. +func (it *Iterator) Next() bool { + it.block = nil + return it.inner.Next() +} + +// Number is the number of the block currently loaded. +func (it *Iterator) Number() uint64 { return it.inner.next - 1 } + +// Error returns any iteration error (EOF is reported as nil, identical +// to the Era‑1 iterator behaviour). +func (it *Iterator) Error() error { return it.inner.Error() } + +// Block decodes the current header+body into a *types.Block. +func (it *Iterator) Block() (*types.Block, error) { + if it.block != nil { + return it.block, nil + } + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non‑nil") + } + var ( + h types.Header + b types.Body + ) + if err := rlp.Decode(it.inner.Header, &h); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &b); err != nil { + return nil, err + } + it.block = types.NewBlockWithHeader(&h).WithBody(b) + return it.block, nil +} + +// Receipts decodes receipts for the current block. +func (it *Iterator) Receipts() (types.Receipts, error) { + block, err := it.Block() + if err != nil { + return nil, err + } + if it.inner.Receipts == nil { + return nil, errors.New("receipts must be non‑nil") + } + var rs []*types.SlimReceipt + if err := rlp.Decode(it.inner.Receipts, &rs); err != nil { + return nil, err + } + if len(rs) != len(block.Transactions()) { + return nil, errors.New("number of txs does not match receipts") + } + receipts := make([]*types.Receipt, len(rs)) + for i, receipt := range rs { + receipts[i] = (*types.Receipt)(receipt) + receipts[i].Bloom = types.CreateBloom(receipts[i]) + } + return receipts, nil +} + +// BlockAndReceipts is a convenience wrapper. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the TD at the current position (if present). +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + if it.inner.TotalDifficulty == nil { + return nil, errors.New("total‑difficulty stream is nil") + } + tdBytes, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + slices.Reverse(tdBytes) + return new(big.Int).SetBytes(tdBytes), nil +} + +// ----------------------------------------------------------------------------- +// Low‑level iterator (raw TLV/offset handling, no decoding) +// ----------------------------------------------------------------------------- + +type RawIterator struct { + e *Era + next uint64 // next block to pull + err error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader // nil when archive omits TD +} + +// NewRawIterator creates an iterator positioned *before* the first block. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{e: e, next: e.m.start}, nil +} + +// Next loads the next block’s components; returns false on EOF or error. +func (it *RawIterator) Next() bool { + it.err = nil // clear previous error + + if it.next >= it.e.m.start+it.e.m.count { + it.clear() + return false + } + + headerOffset, err := it.e.headerOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Header, _, err = newSnappyReader(it.e.s, era.TypeCompressedHeader, headerOffset) + if err != nil { + it.setErr(err) + return false + } + + bodyOffset, err := it.e.bodyOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Body, _, err = newSnappyReader(it.e.s, era.TypeCompressedBody, bodyOffset) + if err != nil { + it.setErr(err) + return false + } + + receiptsOffset, err := it.e.receiptOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedSlimReceipts, receiptsOffset) + if err != nil { + it.setErr(err) + return false + } + + // Check if TD component is present in this file (pre-merge or merge-transition epoch). + if int(td) < int(it.e.m.components) { + tdOffset, err := it.e.tdOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.TotalDifficulty, _, err = it.e.s.ReaderAt(era.TypeTotalDifficulty, tdOffset) + if err != nil { + it.setErr(err) + return false + } + } else { + it.TotalDifficulty = nil + } + + it.next++ + return true +} + +func (it *RawIterator) Number() uint64 { return it.next - 1 } + +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +func (it *RawIterator) setErr(err error) { + it.err = err + it.clear() +} + +func (it *RawIterator) clear() { + it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil +} + +// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader +// plus the length of the underlying TLV payload so callers can advance offsets. +func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) { + raw, n, err := r.ReaderAt(typ, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(raw), int64(n), nil +} diff --git a/internal/era/execdb/reader.go b/internal/era/execdb/reader.go new file mode 100644 index 0000000000..d0aaad1748 --- /dev/null +++ b/internal/era/execdb/reader.go @@ -0,0 +1,296 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +// Era object represents an era file that contains blocks and their components. +type Era struct { + f era.ReadAtSeekCloser + s *e2store.Reader + m metadata // metadata for the Era file +} + +// Filename returns a recognizable filename for an EraE file. +// The filename uses the last block hash to uniquely identify the epoch's content. +func Filename(network string, epoch int, lastBlockHash common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.erae", network, epoch, lastBlockHash.Hex()[2:10]) +} + +// Open accesses the era file. +func Open(path string) (*Era, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Close closes the era file safely. +func (e *Era) Close() error { + if e.f == nil { + return nil + } + err := e.f.Close() + e.f = nil + return err +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Start retrieves the starting block number. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count retrieves the count of blocks present. +func (e *Era) Count() uint64 { + return e.m.count +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber retrieves the block if present within the era file. +func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) { + h, err := e.GetHeader(blockNum) + if err != nil { + return nil, err + } + b, err := e.GetBody(blockNum) + if err != nil { + return nil, err + } + return types.NewBlockWithHeader(h).WithBody(*b), nil +} + +// GetHeader retrieves the header from the era file through the cached offset table. +func (e *Era) GetHeader(num uint64) (*types.Header, error) { + off, err := e.headerOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var h types.Header + return &h, rlp.Decode(r, &h) +} + +// GetBody retrieves the body from the era file through cached offset table. +func (e *Era) GetBody(num uint64) (*types.Body, error) { + off, err := e.bodyOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var b types.Body + return &b, rlp.Decode(r, &b) +} + +// GetTD retrieves the td from the era file through cached offset table. +func (e *Era) GetTD(blockNum uint64) (*big.Int, error) { + off, err := e.tdOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeTotalDifficulty, off) + if err != nil { + return nil, err + } + buf, _ := io.ReadAll(r) + slices.Reverse(buf) + td := new(big.Int).SetBytes(buf) + return td, nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(blockNum uint64) ([]byte, error) { + off, err := e.bodyOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(blockNum uint64) ([]byte, error) { + off, err := e.receiptOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era is applied. Returns an error if TD is not available +// (e.g., post-merge epoch). +func (e *Era) InitialTD() (*big.Int, error) { + // Check if TD component exists. + if int(td) >= int(e.m.components) { + return nil, fmt.Errorf("total difficulty not available in this epoch") + } + + // Get first header to read its difficulty. + header, err := e.GetHeader(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first header: %w", err) + } + + // Get TD after first block using the index. + firstTD, err := e.GetTD(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first TD: %w", err) + } + + // Initial TD = TD[0] - Difficulty[0] + return new(big.Int).Sub(firstTD, header.Difficulty), nil +} + +// Accumulator reads the accumulator entry in the EraE file if it exists. +// Note that one premerge erae files will contain an accumulator entry. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// loadIndex loads in the index table containing all offsets and caches it. +func (e *Era) loadIndex() error { + var err error + e.m.length, err = e.f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + + b := make([]byte, 16) + if _, err = e.f.ReadAt(b, e.m.length-16); err != nil { + return err + } + e.m.components = binary.LittleEndian.Uint64(b[0:8]) + e.m.count = binary.LittleEndian.Uint64(b[8:16]) + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + tlvstart := e.m.length - int64(payloadlen) - 8 + _, err = e.f.ReadAt(b[:8], tlvstart+8) + if err != nil { + return err + } + + e.m.start = binary.LittleEndian.Uint64(b[:8]) + return nil +} + +// headerOff, bodyOff, receiptOff, and tdOff return the offsets of the respective components for a given block number. +func (e *Era) headerOff(num uint64) (int64, error) { return e.indexOffset(num, header) } +func (e *Era) bodyOff(num uint64) (int64, error) { return e.indexOffset(num, body) } +func (e *Era) receiptOff(num uint64) (int64, error) { return e.indexOffset(num, receipts) } +func (e *Era) tdOff(num uint64) (int64, error) { return e.indexOffset(num, td) } + +// indexOffset calculates offset to a certain component for a block number within a file. +func (e *Era) indexOffset(n uint64, component componentType) (int64, error) { + if n < e.m.start || n >= e.m.start+e.m.count { + return 0, fmt.Errorf("block %d out of range [%d,%d)", n, e.m.start, e.m.start+e.m.count) + } + if int(component) >= int(e.m.components) { + return 0, fmt.Errorf("component %d not present", component) + } + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + indstart := e.m.length - int64(payloadlen) - 8 + + rec := (n-e.m.start)*e.m.components + uint64(component) + pos := indstart + 8 + 8 + int64(rec*8) + + var buf [8]byte + if _, err := e.f.ReadAt(buf[:], pos); err != nil { + return 0, err + } + rel := binary.LittleEndian.Uint64(buf[:]) + return int64(rel) + indstart, nil +} + +// metadata contains the information about the era file that is written into the file. +type metadata struct { + start uint64 // start block number + count uint64 // number of blocks in the era + components uint64 // number of properties + length int64 // length of the file in bytes +} + +// componentType represents the integer form of a specific type that can be present in the era file. +type componentType int + +// header, body, receipts, td, and proof are the different types of components that can be present in the era file. +const ( + header componentType = iota + body + receipts + td + proof +) diff --git a/internal/era/builder.go b/internal/era/onedb/builder.go similarity index 84% rename from internal/era/builder.go rename to internal/era/onedb/builder.go index 975561564c..6497bf4f60 100644 --- a/internal/era/builder.go +++ b/internal/era/onedb/builder.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/e2store" "github.com/ethereum/go-ethereum/rlp" "github.com/golang/snappy" @@ -72,20 +73,22 @@ import ( // Due to the accumulator size limit of 8192, the maximum number of blocks in // an Era1 batch is also 8192. type Builder struct { - w *e2store.Writer - startNum *uint64 - startTd *big.Int - indexes []uint64 - hashes []common.Hash - tds []*big.Int - written int + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + accumulator *common.Hash // accumulator root, set by Finalize + + written int buf *bytes.Buffer snappy *snappy.Writer } // NewBuilder returns a new Builder instance. -func NewBuilder(w io.Writer) *Builder { +func NewBuilder(w io.Writer) era.Builder { buf := bytes.NewBuffer(nil) return &Builder{ w: e2store.NewWriter(w), @@ -117,7 +120,7 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { // Write Era1 version entry before first block. if b.startNum == nil { - n, err := b.w.Write(TypeVersion, nil) + n, err := b.w.Write(era.TypeVersion, nil) if err != nil { return err } @@ -126,8 +129,8 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.startTd = new(big.Int).Sub(td, difficulty) b.written += n } - if len(b.indexes) >= MaxEra1Size { - return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + if len(b.indexes) >= era.MaxSize { + return fmt.Errorf("exceeds maximum batch size of %d", era.MaxSize) } b.indexes = append(b.indexes, uint64(b.written)) @@ -135,19 +138,19 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.tds = append(b.tds, td) // Write block data. - if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + if err := b.snappyWrite(era.TypeCompressedHeader, header); err != nil { return err } - if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + if err := b.snappyWrite(era.TypeCompressedBody, body); err != nil { return err } - if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + if err := b.snappyWrite(era.TypeCompressedReceipts, receipts); err != nil { return err } // Also write total difficulty, but don't snappy encode. - btd := bigToBytes32(td) - n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + btd := era.BigToBytes32(td) + n, err := b.w.Write(era.TypeTotalDifficulty, btd[:]) b.written += n if err != nil { return err @@ -157,21 +160,24 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm } // Finalize computes the accumulator and block index values, then writes the -// corresponding e2store entries. +// corresponding e2store entries. Era1 always has an accumulator, so this +// always returns a valid hash. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("finalize called on empty builder") } // Compute accumulator root and write entry. - root, err := ComputeAccumulator(b.hashes, b.tds) + root, err := era.ComputeAccumulator(b.hashes, b.tds) if err != nil { return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) } - n, err := b.w.Write(TypeAccumulator, root[:]) + n, err := b.w.Write(era.TypeAccumulator, root[:]) b.written += n if err != nil { return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) } + b.accumulator = &root + // Get beginning of index entry to calculate block relative offset. base := int64(b.written) @@ -196,13 +202,19 @@ func (b *Builder) Finalize() (common.Hash, error) { binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) // Finally, write the block index entry. - if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + if _, err := b.w.Write(era.TypeBlockIndex, index); err != nil { return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) } return root, nil } +// Accumulator returns the accumulator root after Finalize has been called. +// For Era1, this always returns a non-nil value since all blocks are pre-merge. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + // snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. func (b *Builder) snappyWrite(typ uint16, in []byte) error { var ( diff --git a/internal/era/era_test.go b/internal/era/onedb/builder_test.go similarity index 95% rename from internal/era/era_test.go rename to internal/era/onedb/builder_test.go index 31fa0076a6..bc7a1d9e63 100644 --- a/internal/era/era_test.go +++ b/internal/era/onedb/builder_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -22,6 +22,7 @@ import ( "io" "math/big" "os" + "slices" "testing" "github.com/ethereum/go-ethereum/common" @@ -82,7 +83,11 @@ func TestEra1Builder(t *testing.T) { t.Fatalf("failed to open era: %v", err) } defer e.Close() - it, err := NewRawIterator(e) + eraPtr, ok := e.(*Era) + if !ok { + t.Fatalf("failed to assert *Era type") + } + it, err := NewRawIterator(eraPtr) if err != nil { t.Fatalf("failed to make iterator: %s", err) } @@ -119,7 +124,7 @@ func TestEra1Builder(t *testing.T) { if !bytes.Equal(rawReceipts, chain.receipts[i]) { t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts) } - receipts, err := getReceiptsByNumber(e, i) + receipts, err := getReceiptsByNumber(eraPtr, i) if err != nil { t.Fatalf("error reading receipts: %v", err) } @@ -136,7 +141,8 @@ func TestEra1Builder(t *testing.T) { if err != nil { t.Fatalf("error reading td: %v", err) } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) if td.Cmp(chain.tds[i]) != 0 { t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) } diff --git a/internal/era/iterator.go b/internal/era/onedb/iterator.go similarity index 89% rename from internal/era/iterator.go rename to internal/era/onedb/iterator.go index 3c4f82d850..b80fbabbc5 100644 --- a/internal/era/iterator.go +++ b/internal/era/onedb/iterator.go @@ -14,14 +14,16 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "errors" "io" "math/big" + "slices" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/rlp" ) @@ -32,8 +34,8 @@ type Iterator struct { // NewIterator returns a new Iterator instance. Next must be immediately // called on new iterators to load the first item. -func NewIterator(e *Era) (*Iterator, error) { - inner, err := NewRawIterator(e) +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) if err != nil { return nil, err } @@ -107,7 +109,8 @@ func (it *Iterator) TotalDifficulty() (*big.Int, error) { if err != nil { return nil, err } - return new(big.Int).SetBytes(reverseOrder(td)), nil + slices.Reverse(td) + return new(big.Int).SetBytes(td), nil } // RawIterator reads an RLP-encode Era1 entries. @@ -151,22 +154,22 @@ func (it *RawIterator) Next() bool { return false } var n int64 - if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + if it.Header, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedHeader, off); it.err != nil { it.clear() return true } off += n - if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + if it.Body, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedBody, off); it.err != nil { it.clear() return true } off += n - if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + if it.Receipts, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, off); it.err != nil { it.clear() return true } off += n - if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(era.TypeTotalDifficulty, off); it.err != nil { it.clear() return true } diff --git a/internal/era/onedb/reader.go b/internal/era/onedb/reader.go new file mode 100644 index 0000000000..df93ca8211 --- /dev/null +++ b/internal/era/onedb/reader.go @@ -0,0 +1,279 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package onedb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (era.Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber returns the block for the given block number. +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body), nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + off, err = e.s.SkipN(off, 1) + if err != nil { + return nil, err + } + r, _, err := newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + r, _, err := newSnappyReader(e.s, era.TypeCompressedReceipts, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, era.TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(era.TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clear(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newSnappyReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/eth/protocols/snap/tracker.go b/internal/era/proof.go similarity index 66% rename from eth/protocols/snap/tracker.go rename to internal/era/proof.go index 2cf59cc23a..464d9e6fe5 100644 --- a/eth/protocols/snap/tracker.go +++ b/internal/era/proof.go @@ -1,4 +1,4 @@ -// Copyright 2021 The go-ethereum Authors +// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -13,14 +13,24 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . - -package snap +package era import ( - "time" + "io" + + "github.com/ethereum/go-ethereum/rlp" +) + +type ProofVariant uint16 - "github.com/ethereum/go-ethereum/p2p/tracker" +const ( + ProofNone ProofVariant = iota ) -// requestTracker is a singleton tracker for request times. -var requestTracker = tracker.New(ProtocolName, time.Minute) +// Proof is the interface for all block proof types in the package. +// It's a stub for later integration into Era. +type Proof interface { + EncodeRLP(w io.Writer) error + DecodeRLP(s *rlp.Stream) error + Variant() ProofVariant +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0b2e3ad4dc..cb54330ded 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -49,7 +49,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" ) var errBlobTxNotSupported = errors.New("signing blob transactions not supported") @@ -179,6 +178,15 @@ func NewTxPoolAPI(b Backend) *TxPoolAPI { return &TxPoolAPI{b} } +// flattenTxs builds the RPC transaction map keyed by nonce for a set of pool txs. +func flattenTxs(txs types.Transactions, header *types.Header, cfg *params.ChainConfig) map[string]*RPCTransaction { + dump := make(map[string]*RPCTransaction, len(txs)) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, header, cfg) + } + return dump +} + // Content returns the transactions contained within the transaction pool. func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { pending, queue := api.b.TxPoolContent() @@ -189,19 +197,11 @@ func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction curHeader := api.b.CurrentHeader() // Flatten the pending transactions for account, txs := range pending { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["pending"][account.Hex()] = dump + content["pending"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } // Flatten the queued transactions for account, txs := range queue { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["queued"][account.Hex()] = dump + content["queued"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } return content } @@ -213,18 +213,10 @@ func (api *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RP curHeader := api.b.CurrentHeader() // Build the pending transactions - dump := make(map[string]*RPCTransaction, len(pending)) - for _, tx := range pending { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["pending"] = dump + content["pending"] = flattenTxs(pending, curHeader, api.b.ChainConfig()) // Build the queued transactions - dump = make(map[string]*RPCTransaction, len(queue)) - for _, tx := range queue { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - content["queued"] = dump + content["queued"] = flattenTxs(queue, curHeader, api.b.ChainConfig()) return content } @@ -372,9 +364,9 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, // Deserialize all keys. This prevents state access on invalid input. for i, hexKey := range storageKeys { var err error - keys[i], keyLengths[i], err = decodeHash(hexKey) + keys[i], keyLengths[i], err = decodeStorageKey(hexKey) if err != nil { - return nil, err + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } } statedb, header, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) @@ -387,8 +379,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, if len(keys) > 0 { var storageTrie state.Trie if storageRoot != types.EmptyRootHash && storageRoot != (common.Hash{}) { - id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) - st, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) + st, err := statedb.Database().OpenStorageTrie(header.Root, address, storageRoot, nil) if err != nil { return nil, err } @@ -419,7 +410,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, } } // Create the accountProof. - tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), statedb.Database().TrieDB()) + tr, err := statedb.Database().OpenTrie(header.Root) if err != nil { return nil, err } @@ -439,9 +430,10 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, }, statedb.Error() } -// decodeHash parses a hex-encoded 32-byte hash. The input may optionally -// be prefixed by 0x and can have a byte length up to 32. -func decodeHash(s string) (h common.Hash, inputLength int, err error) { +// decodeStorageKey parses a hex-encoded 32-byte hash. +// For legacy compatibility reasons, we parse these keys leniently, +// with the 0x prefix being optional. +func decodeStorageKey(s string) (h common.Hash, inputLength int, err error) { if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { s = s[2:] } @@ -449,11 +441,11 @@ func decodeHash(s string) (h common.Hash, inputLength int, err error) { s = "0" + s } if len(s) > 64 { - return common.Hash{}, len(s) / 2, errors.New("hex string too long, want at most 32 bytes") + return common.Hash{}, len(s) / 2, errors.New("storage key too long (want at most 32 bytes)") } b, err := hex.DecodeString(s) if err != nil { - return common.Hash{}, 0, errors.New("hex string invalid") + return common.Hash{}, 0, errors.New("invalid hex in storage key") } return common.BytesToHash(b), len(b), nil } @@ -601,9 +593,9 @@ func (api *BlockChainAPI) GetCode(ctx context.Context, address common.Address, b // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, hexKey string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - key, _, err := decodeHash(hexKey) + key, _, err := decodeStorageKey(hexKey) if err != nil { - return nil, fmt.Errorf("unable to decode storage key: %s", err) + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } state, _, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { @@ -1139,6 +1131,7 @@ func (api *BlockChainAPI) arbClassicL1BlockNumber(ctx context.Context, block *ty type RPCTransaction struct { BlockHash *common.Hash `json:"blockHash"` BlockNumber *hexutil.Big `json:"blockNumber"` + BlockTimestamp *hexutil.Uint64 `json:"blockTimestamp"` From common.Address `json:"from"` Gas hexutil.Uint64 `json:"gas"` GasPrice *hexutil.Big `json:"gasPrice"` @@ -1199,6 +1192,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber if blockHash != (common.Hash{}) { result.BlockHash = &blockHash result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) + result.BlockTimestamp = (*hexutil.Uint64)(&blockTime) result.TransactionIndex = (*hexutil.Uint64)(&index) } @@ -1866,7 +1860,7 @@ func (api *TransactionAPI) SendTransaction(ctx context.Context, args Transaction return common.Hash{}, err } // Assemble the transaction and sign with the wallet - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) signed, err := wallet.SignTx(account, tx, api.b.ChainConfig().ChainID) if err != nil { @@ -1888,7 +1882,7 @@ func (api *TransactionAPI) FillTransaction(ctx context.Context, args Transaction return nil, err } // Assemble the transaction and obtain rlp - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) data, err := tx.MarshalBinary() if err != nil { return nil, err @@ -1929,7 +1923,7 @@ func (api *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil // SendRawTransactionSync will add the signed transaction to the transaction pool // and wait until the transaction has been included in a block and return the receipt, or the timeout. -func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *hexutil.Uint64) (map[string]interface{}, error) { +func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *uint64) (map[string]interface{}, error) { tx := new(types.Transaction) if err := tx.UnmarshalBinary(input); err != nil { return nil, err @@ -2095,7 +2089,7 @@ func (api *TransactionAPI) SignTransaction(ctx context.Context, args Transaction return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) if err := checkTxFee(tx.GasPrice(), tx.Gas(), api.b.RPCTxFeeCap()); err != nil { return nil, err } @@ -2151,7 +2145,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if err := sendArgs.setDefaults(ctx, api.b, sidecarConfig{}); err != nil { return common.Hash{}, err } - matchTx := sendArgs.ToTransaction(types.LegacyTxType) + matchTx := sendArgs.ToTransaction(types.DynamicFeeTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. price := matchTx.GasPrice() @@ -2181,7 +2175,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if gasLimit != nil && *gasLimit != 0 { sendArgs.Gas = gasLimit } - signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.LegacyTxType)) + signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.DynamicFeeTxType)) if err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 0b62dd8c44..7bb3aec7b0 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -141,6 +141,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -171,6 +172,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -209,6 +211,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -255,6 +258,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -302,6 +306,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -346,6 +351,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -388,6 +394,7 @@ func allBlobTxs(addr common.Address, config *params.ChainConfig) []txData { Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x6", "gasPrice": "0x5", @@ -3159,6 +3166,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3179,6 +3187,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3197,6 +3206,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3217,6 +3227,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -4048,7 +4059,7 @@ func TestSendRawTransactionSync_Timeout(t *testing.T) { raw, _ := makeSelfSignedRaw(t, api, b.acc.Address) - timeout := hexutil.Uint64(200) // 200ms + timeout := uint64(200) // 200ms receipt, err := api.SendRawTransactionSync(context.Background(), raw, &timeout) if receipt != nil { diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go index e8e1393bc7..749ce1b989 100644 --- a/internal/ethapi/errors.go +++ b/internal/ethapi/errors.go @@ -120,7 +120,6 @@ const ( errCodeClientLimitExceeded = -38026 errCodeInternalError = -32603 errCodeInvalidParams = -32602 - errCodeReverted = -32000 errCodeVMError = -32015 errCodeTxSyncTimeout = 4 ) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 322e7c06bc..c28a9fe58b 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -224,7 +224,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } arbOSVersion := types.DeserializeHeaderExtraInformation(header).ArbOSFormatVersion parentArbOSVersion := types.DeserializeHeaderExtraInformation(parent).ArbOSFormatVersion - if sim.chainConfig.IsCancun(header.Number, header.Time, arbOSVersion) { + if !sim.chainConfig.IsArbitrum() && sim.chainConfig.IsCancun(header.Number, header.Time, arbOSVersion) { var excess uint64 if sim.chainConfig.IsCancun(parent.Number, parent.Time, parentArbOSVersion) { excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent, header.Time) @@ -235,7 +235,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if block.BlockOverrides.BlobBaseFee != nil { blockContext.BlobBaseFee = block.BlockOverrides.BlobBaseFee.ToInt() } - precompiles := sim.activePrecompiles(sim.base) + precompiles := sim.activePrecompiles(header) // State overrides are applied prior to execution of a block if err := block.StateOverrides.Apply(sim.state, precompiles); err != nil { return nil, nil, nil, err @@ -312,7 +312,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if errors.Is(result.Err, vm.ErrExecutionReverted) { // If the result contains a revert reason, try to unpack it. revertErr := newRevertError(result.Revert()) - callRes.Error = &callError{Message: revertErr.Error(), Code: errCodeReverted, Data: revertErr.ErrorData().(string)} + callRes.Error = &callError{Message: revertErr.Error(), Code: revertErr.ErrorCode(), Data: revertErr.ErrorData().(string)} } else { callRes.Error = &callError{Message: result.Err.Error(), Code: errCodeVMError} } @@ -379,7 +379,7 @@ func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, head call.Gas = (*hexutil.Uint64)(&remaining) } if *gasUsed+uint64(*call.Gas) > blockContext.GasLimit { - return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: %d >= %d", gasUsed, blockContext.GasLimit)} + return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: %d >= %d", *gasUsed, blockContext.GasLimit)} } if err := call.CallDefaults(sim.gp.Gas(), header.BaseFee, sim.chainConfig.ChainID); err != nil { return err diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json index 3c8d42c9a9..6aac5b682e 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json index 3c8d42c9a9..6aac5b682e 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json @@ -20,6 +20,7 @@ { "blockHash": "0xedb9ccf3a85f67c095ad48abfb0fa09d47179bb0f902078d289042d12428aca5", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json index 2e323dcfe7..fc60111244 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xfda6c7cb7a3a712e0c424909a7724cab0448e89e286617fa8d5fd27f63f28bd2", "blockNumber": "0xb", + "blockTimestamp": "0x6e", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0xde56ab3", diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index fd25c17220..6448528c06 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -336,8 +336,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca commitments = make([]kzg4844.Commitment, n) proofs = make([]kzg4844.Proof, 0, proofLen) ) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } @@ -345,13 +345,13 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - p, err := kzg4844.ComputeBlobProof(&b, c) + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], c) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } proofs = append(proofs, p) case types.BlobSidecarVersion1: - ps, err := kzg4844.ComputeCellProofs(&b) + ps, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } @@ -363,8 +363,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca } else { switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index fc84ae85da..e6a6966d9f 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -40,7 +40,7 @@ func NewApp(usage string) *cli.App { app.EnableBashCompletion = true app.Version = version.WithCommit(git.Commit, git.Date) app.Usage = usage - app.Copyright = "Copyright 2013-2025 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2026 The go-ethereum Authors" app.Before = func(ctx *cli.Context) error { MigrateGlobalFlags(ctx) return nil diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000000..27fe9b0a7a --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,109 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package telemetry + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.38.0" + "go.opentelemetry.io/otel/trace" +) + +// Attribute is an alias for attribute.KeyValue. +type Attribute = attribute.KeyValue + +// StringAttribute creates an attribute with a string value. +func StringAttribute(key, val string) Attribute { + return attribute.String(key, val) +} + +// Int64Attribute creates an attribute with an int64 value. +func Int64Attribute(key string, val int64) Attribute { + return attribute.Int64(key, val) +} + +// BoolAttribute creates an attribute with a bool value. +func BoolAttribute(key string, val bool) Attribute { + return attribute.Bool(key, val) +} + +// StartSpan creates a SpanKind=INTERNAL span. +func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + return StartSpanWithTracer(ctx, otel.Tracer(""), spanName, attributes...) +} + +// StartSpanWithTracer requires a tracer to be passed in and creates a SpanKind=INTERNAL span. +func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + // Don't create a span if there's no parent span in the context. + parent := trace.SpanFromContext(ctx) + if !parent.SpanContext().IsValid() { + return ctx, parent, func(*error) {} + } + return startSpan(ctx, tracer, trace.SpanKindInternal, name, attributes...) +} + +// RPCInfo contains information about the RPC request. +type RPCInfo struct { + System string + Service string + Method string + RequestID string +} + +// StartServerSpan creates a SpanKind=SERVER span at the JSON-RPC boundary. +// The span name is formatted as $rpcSystem.$rpcService/$rpcMethod +// (e.g. "jsonrpc.engine/newPayloadV4") which follows the Open Telemetry +// semantic convensions: https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#span-name. +func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, others ...Attribute) (context.Context, func(*error)) { + var ( + name = fmt.Sprintf("%s.%s/%s", rpc.System, rpc.Service, rpc.Method) + attributes = append([]Attribute{ + semconv.RPCSystemKey.String(rpc.System), + semconv.RPCServiceKey.String(rpc.Service), + semconv.RPCMethodKey.String(rpc.Method), + semconv.RPCJSONRPCRequestID(rpc.RequestID), + }, + others..., + ) + ) + ctx, _, end := startSpan(ctx, tracer, trace.SpanKindServer, name, attributes...) + return ctx, end +} + +// startSpan creates a span with the given kind. +func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(kind)) + if len(attributes) > 0 { + span.SetAttributes(attributes...) + } + return ctx, span, endSpan(span) +} + +// endSpan ends the span and handles error recording. +func endSpan(span trace.Span) func(*error) { + return func(err *error) { + if err != nil && *err != nil { + span.RecordError(*err) + span.SetStatus(codes.Error, (*err).Error()) + } + span.End() + } +} diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000000..def85b735e --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package telemetry + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// newTestTracer creates a TracerProvider backed by an in-memory exporter. +func newTestTracer(t *testing.T) (trace.Tracer, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + return tp.Tracer("test"), tp, exporter +} + +func TestStartSpanWithTracer_NoParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a span without a parent. + ctx := context.Background() + retCtx, _, endSpan := StartSpanWithTracer(ctx, tracer, "should-not-exist") + endSpan(nil) + + // The returned context should be the original context (unchanged). + if retCtx != ctx { + t.Fatal("expected original context to be returned unchanged") + } + + // Flush and verify no spans were recorded. + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) != 0 { + t.Fatalf("expected no spans, got %d", len(spans)) + } +} + +func TestStartSpanWithTracer_WithParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a parent span to establish a valid span context. + ctx, parentSpan := tracer.Start(context.Background(), "parent") + defer parentSpan.End() + + // Should create a real child span. + _, _, endSpan := StartSpanWithTracer(ctx, tracer, "child") + endSpan(nil) + + // Flush and verify the child span was recorded. + parentSpan.End() + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + var childSpan *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "child" { + childSpan = &spans[i] + break + } + } + if childSpan == nil { + t.Fatal("child span not found") + } + + // Verify it is parented to the correct trace. + if childSpan.Parent.TraceID() != parentSpan.SpanContext().TraceID() { + t.Errorf("trace ID mismatch: got %s, want %s", + childSpan.Parent.TraceID(), parentSpan.SpanContext().TraceID()) + } + if childSpan.SpanKind != trace.SpanKindInternal { + t.Errorf("expected SpanKindInternal, got %v", childSpan.SpanKind) + } +} diff --git a/internal/telemetry/tracesetup/setup.go b/internal/telemetry/tracesetup/setup.go new file mode 100644 index 0000000000..9637ca1a9b --- /dev/null +++ b/internal/telemetry/tracesetup/setup.go @@ -0,0 +1,162 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracesetup + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "strings" + "time" + + "github.com/ethereum/go-ethereum/internal/version" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.38.0" +) + +const startStopTimeout = 10 * time.Second + +// Service wraps the provider to implement node.Lifecycle. +type Service struct { + endpoint string + exporter *otlptrace.Exporter + provider *sdktrace.TracerProvider +} + +// Start implements node.Lifecycle. +func (t *Service) Start() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.exporter.Start(ctx); err != nil { + log.Error("OpenTelemetry exporter didn't start", "endpoint", t.endpoint, "err", err) + return err + } + log.Info("OpenTelemetry trace export enabled", "endpoint", t.endpoint) + return nil +} + +// Stop implements node.Lifecycle. +func (t *Service) Stop() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.provider.Shutdown(ctx); err != nil { + log.Error("Failed to stop OpenTelemetry service", "err", err) + return err + } + log.Debug("OpenTelemetry stopped") + return nil +} + +// SetupTelemetry initializes telemetry with the given parameters. +func SetupTelemetry(cfg node.OpenTelemetryConfig, stack *node.Node) error { + if !cfg.Enabled { + return nil + } + if cfg.SampleRatio < 0 || cfg.SampleRatio > 1 { + return fmt.Errorf("invalid sample ratio: %f", cfg.SampleRatio) + } + // Create exporter based on endpoint URL + u, err := url.Parse(cfg.Endpoint) + if err != nil { + return fmt.Errorf("invalid rpc tracing endpoint URL: %w", err) + } + var exporter *otlptrace.Exporter + switch u.Scheme { + case "http", "https": + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(u.Host), + } + if u.Scheme == "http" { + opts = append(opts, otlptracehttp.WithInsecure()) + } + if u.Path != "" && u.Path != "/" { + opts = append(opts, otlptracehttp.WithURLPath(u.Path)) + } + if cfg.AuthUser != "" { + opts = append(opts, otlptracehttp.WithHeaders(map[string]string{ + "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(cfg.AuthUser+":"+cfg.AuthPassword)), + })) + } + exporter = otlptracehttp.NewUnstarted(opts...) + default: + return fmt.Errorf("unsupported telemetry url scheme: %s", u.Scheme) + } + + // Define sampler such that if no parent span is available, + // then sampleRatio of traces are sampled; otherwise, inherit + // the parent's sampling decision. + sampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio)) + + // Define batch span processor options + batchOpts := []sdktrace.BatchSpanProcessorOption{ + // The maximum number of spans that can be queued before dropping + sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxExportBatchSize), + // The maximum number of spans to export in a single batch + sdktrace.WithMaxExportBatchSize(sdktrace.DefaultMaxExportBatchSize), + // How long an export operation can take before timing out + sdktrace.WithExportTimeout(time.Duration(sdktrace.DefaultExportTimeout) * time.Millisecond), + // How often to export, even if the batch isn't full + sdktrace.WithBatchTimeout(time.Duration(sdktrace.DefaultScheduleDelay) * time.Millisecond), + } + + // Define resource attributes + var attr = []attribute.KeyValue{ + semconv.ServiceName("geth"), + attribute.String("client.name", version.ClientName("geth")), + } + // Add instance ID if provided + if cfg.InstanceID != "" { + attr = append(attr, semconv.ServiceInstanceID(cfg.InstanceID)) + } + // Add custom tags if provided + if cfg.Tags != "" { + for tag := range strings.SplitSeq(cfg.Tags, ",") { + key, value, ok := strings.Cut(tag, "=") + if ok { + attr = append(attr, attribute.String(key, value)) + } + } + } + res := resource.NewWithAttributes(semconv.SchemaURL, attr...) + + // Configure TracerProvider and set it as the global tracer provider + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sampler), + sdktrace.WithBatcher(exporter, batchOpts...), + sdktrace.WithResource(res), + ) + otel.SetTracerProvider(tp) + + // Set global propagator for context propagation + // Note: This is needed for distributed tracing + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + service := &Service{endpoint: cfg.Endpoint, exporter: exporter, provider: tp} + stack.RegisterLifecycle(service) + return nil +} diff --git a/metrics/counter.go b/metrics/counter.go index c884e9a178..0e4d93bfbc 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -7,7 +7,10 @@ import ( // GetOrRegisterCounter returns an existing Counter or constructs and registers // a new Counter. func GetOrRegisterCounter(name string, r Registry) *Counter { - return getOrRegister(name, NewCounter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounter() }).(*Counter) } // NewCounter constructs a new Counter. diff --git a/metrics/counter_float64.go b/metrics/counter_float64.go index 6cc73d89a2..caaaeb7be7 100644 --- a/metrics/counter_float64.go +++ b/metrics/counter_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers // a new CounterFloat64. func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { - return getOrRegister(name, NewCounterFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounterFloat64() }).(*CounterFloat64) } // NewCounterFloat64 constructs a new CounterFloat64. diff --git a/metrics/gauge.go b/metrics/gauge.go index 20de95255b..39d5ccb3bc 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -11,7 +11,10 @@ func (g GaugeSnapshot) Value() int64 { return int64(g) } // GetOrRegisterGauge returns an existing Gauge or constructs and registers a // new Gauge. func GetOrRegisterGauge(name string, r Registry) *Gauge { - return getOrRegister(name, NewGauge, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGauge() }).(*Gauge) } // NewGauge constructs a new Gauge. diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 48524e4c3f..c7a1df0a23 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a // new GaugeFloat64. func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { - return getOrRegister(name, NewGaugeFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeFloat64() }).(*GaugeFloat64) } // GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index 34ac917919..30c5114085 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -16,7 +16,10 @@ func (val GaugeInfoValue) String() string { // GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a // new GaugeInfo. func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { - return getOrRegister(name, NewGaugeInfo, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeInfo() }).(*GaugeInfo) } // NewGaugeInfo constructs a new GaugeInfo. diff --git a/metrics/histogram.go b/metrics/histogram.go index 18bf6e3d2b..467457abdb 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -23,13 +23,19 @@ type Histogram interface { // GetOrRegisterHistogram returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s) }).(Histogram) } // GetOrRegisterHistogramLazy returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s()) }).(Histogram) } // NewHistogram constructs a new StandardHistogram from a Sample. diff --git a/metrics/meter.go b/metrics/meter.go index ee23af10eb..2829774d8c 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -12,7 +12,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) *Meter { - return getOrRegister(name, NewMeter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewMeter() }).(*Meter) } // NewMeter constructs a new Meter and launches a goroutine. diff --git a/metrics/registry.go b/metrics/registry.go index 6070f3d0e9..298598a9aa 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -143,6 +143,8 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["value"] = metric.Snapshot().Value() case *GaugeFloat64: values["value"] = metric.Snapshot().Value() + case *GaugeInfo: + values["value"] = metric.Snapshot().Value() case *Healthcheck: values["error"] = nil metric.Check() @@ -186,6 +188,18 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["5m.rate"] = t.Rate5() values["15m.rate"] = t.Rate15() values["mean.rate"] = t.RateMean() + case *ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] } data[name] = values }) @@ -331,13 +345,6 @@ func GetOrRegister(name string, i func() interface{}) interface{} { return DefaultRegistry.GetOrRegister(name, i) } -func getOrRegister[T any](name string, ctor func() T, r Registry) T { - if r == nil { - r = DefaultRegistry - } - return r.GetOrRegister(name, func() any { return ctor() }).(T) -} - // Register the given metric under the given name. Returns a ErrDuplicateMetric // if a metric by the given name is already registered. func Register(name string, i interface{}) error { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6af0796da9..1aad7a0028 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -14,6 +14,31 @@ func BenchmarkRegistry(b *testing.B) { } } +func BenchmarkRegistryGetOrRegister(b *testing.B) { + sample := func() Sample { return nil } + tests := []struct { + name string + ctor func() any + }{ + {name: "counter", ctor: func() any { return GetOrRegisterCounter("counter", DefaultRegistry) }}, + {name: "gauge", ctor: func() any { return GetOrRegisterGauge("gauge", DefaultRegistry) }}, + {name: "gaugefloat64", ctor: func() any { return GetOrRegisterGaugeFloat64("gaugefloat64", DefaultRegistry) }}, + {name: "histogram", ctor: func() any { return GetOrRegisterHistogram("histogram", DefaultRegistry, sample()) }}, + {name: "meter", ctor: func() any { return GetOrRegisterMeter("meter", DefaultRegistry) }}, + {name: "timer", ctor: func() any { return GetOrRegisterTimer("timer", DefaultRegistry) }}, + {name: "gaugeinfo", ctor: func() any { return GetOrRegisterGaugeInfo("gaugeinfo", DefaultRegistry) }}, + {name: "resettingtimer", ctor: func() any { return GetOrRegisterResettingTimer("resettingtimer", DefaultRegistry) }}, + {name: "runtimehistogramlazy", ctor: func() any { return GetOrRegisterHistogramLazy("runtimehistogramlazy", DefaultRegistry, sample) }}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + test.ctor() + } + }) + } +} + func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { benchmarkRegistryGetOrRegisterParallel(b, 8) } @@ -268,7 +293,7 @@ func TestPrefixedChildRegistryGet(t *testing.T) { } func TestChildPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") err := r.Register("foo", NewCounter()) c := NewCounter() Register("bar", c) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 8aa7dc1488..a3f46e52e0 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a // new ResettingTimer. func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { - return getOrRegister(name, NewResettingTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewResettingTimer() }).(*ResettingTimer) } // NewRegisteredResettingTimer constructs and registers a new ResettingTimer. diff --git a/metrics/runtimehistogram.go b/metrics/runtimehistogram.go index efbed498af..e975a570a4 100644 --- a/metrics/runtimehistogram.go +++ b/metrics/runtimehistogram.go @@ -8,8 +8,10 @@ import ( ) func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runtimeHistogram { - constructor := func() Histogram { return newRuntimeHistogram(scale) } - return getOrRegister(name, constructor, r).(*runtimeHistogram) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return newRuntimeHistogram(scale) }).(*runtimeHistogram) } // runtimeHistogram wraps a runtime/metrics histogram. diff --git a/metrics/timer.go b/metrics/timer.go index 894bdfc327..8082b31947 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -10,7 +10,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) *Timer { - return getOrRegister(name, NewTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewTimer() }).(*Timer) } // NewCustomTimer constructs a new Timer from a Histogram and a Meter. diff --git a/miner/miner.go b/miner/miner.go index 90d8116203..24601e0652 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -48,6 +48,7 @@ type Config struct { GasCeil uint64 // Target gas ceiling for mined blocks. GasPrice *big.Int // Minimum gas price for mining a transaction Recommit time.Duration // The time interval for miner to re-create mining work. + MaxBlobsPerBlock int // Maximum number of blobs per block (0 for unset uses protocol default) } // DefaultConfig contains default settings for miner. diff --git a/miner/worker.go b/miner/worker.go index 9e54aecdbe..d2a0905e43 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -43,6 +43,16 @@ var ( errBlockInterruptedByTimeout = errors.New("timeout while building block") ) +// maxBlobsPerBlock returns the maximum number of blobs per block. +// Users can specify the maximum number of blobs per block if necessary. +func (miner *Miner) maxBlobsPerBlock(time uint64) int { + maxBlobs := eip4844.MaxBlobsPerBlock(miner.chainConfig, time) + if miner.config.MaxBlobsPerBlock != 0 { + maxBlobs = miner.config.MaxBlobsPerBlock + } + return maxBlobs +} + // environment is the worker's current environment and holds all // information of the sealing block generation. type environment struct { @@ -314,7 +324,7 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio // isn't really a better place right now. The blob gas limit is checked at block validation time // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. - maxBlobs := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) + maxBlobs := miner.maxBlobsPerBlock(env.header.Time) if env.blobs+len(sc.Blobs) > maxBlobs { return errors.New("max data blobs reached") } @@ -370,7 +380,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran } // If we don't have enough blob space for any further blob transactions, // skip that list altogether - if !blobTxs.Empty() && env.blobs >= eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) { + if !blobTxs.Empty() && env.blobs >= miner.maxBlobsPerBlock(env.header.Time) { log.Trace("Not enough blob space for further blob transactions") blobTxs.Clear() // Fall though to pick up any plain txs @@ -409,7 +419,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // blobs or not, however the max check panics when called on a chain without // a defined schedule, so we need to verify it's safe to call. if isCancun { - left := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) - env.blobs + left := miner.maxBlobsPerBlock(env.header.Time) - env.blobs if left < int(ltx.BlobGas/params.BlobTxBlobGasPerBlob) { log.Trace("Not enough blob space left for transaction", "hash", ltx.Hash, "left", left, "needed", ltx.BlobGas/params.BlobTxBlobGasPerBlob) txs.Pop() diff --git a/node/config.go b/node/config.go index dcf0fb44da..1cdebb6583 100644 --- a/node/config.go +++ b/node/config.go @@ -201,9 +201,7 @@ type Config struct { GraphQLVirtualHosts []string `toml:",omitempty"` // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger `toml:",omitempty"` - - oldGethResourceWarning bool + Logger log.Logger `toml:"-,omitempty"` // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. AllowUnprotectedTxs bool `toml:",omitempty"` @@ -220,6 +218,7 @@ type Config struct { // EnablePersonal enables the deprecated personal namespace. EnablePersonal bool `toml:"-"` + // Configures database engine used by the node. DBEngine string `toml:",omitempty"` // HTTPBodyLimit is the maximum number of bytes allowed in the HTTP request body. @@ -227,6 +226,27 @@ type Config struct { // WSReadLimit is the maximum number of bytes allowed in the websocket request body. WSReadLimit int64 `toml:",omitempty"` + + // Configures OpenTelemetry reporting. + OpenTelemetry OpenTelemetryConfig `toml:",omitempty"` + + oldGethResourceWarning bool +} + +// OpenTelemetryConfig has settings for +type OpenTelemetryConfig struct { + Enabled bool `toml:",omitempty"` + + Tags string `toml:",omitempty"` + InstanceID string `toml:",omitempty"` + + // Exporter endpoint. + Endpoint string `toml:",omitempty"` + AuthUser string `toml:",omitempty"` + AuthPassword string `toml:",omitempty"` + + // Percentage of sampled traces. + SampleRatio float64 `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/node/defaults.go b/node/defaults.go index 917b8bc0a2..3af605baaa 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -71,9 +71,11 @@ var DefaultConfig = Config{ BatchResponseMaxSize: 25 * 1000 * 1000, GraphQLVirtualHosts: []string{"localhost"}, P2P: p2p.Config{ - ListenAddr: ":30303", - MaxPeers: 50, - NAT: nat.Any(), + ListenAddr: ":30303", + MaxPeers: 50, + NAT: nat.Any(), + DiscoveryV4: true, + DiscoveryV5: true, }, DBEngine: "", // Use whatever exists, will default to Pebble if non-existent and supported } diff --git a/node/rpcstack.go b/node/rpcstack.go index f9806c7384..53f3303659 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -140,6 +140,9 @@ func (h *httpServer) start() error { // Initialize the server. h.server = &http.Server{Handler: h} + h.server.Protocols = new(http.Protocols) + h.server.Protocols.SetHTTP1(true) + h.server.Protocols.SetUnencryptedHTTP2(true) if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 54e58cccb2..bd75dac4eb 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -593,6 +593,38 @@ func TestHTTPWriteTimeout(t *testing.T) { }) } +func TestHTTP2H2C(t *testing.T) { + srv := createAndStartServer(t, &httpConfig{}, false, &wsConfig{}, nil) + defer srv.stop() + + // Create an HTTP/2 cleartext client. + transport := &http.Transport{} + transport.Protocols = new(http.Protocols) + transport.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: transport} + + body := strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"rpc_modules","params":[]}`) + resp, err := client.Post("http://"+srv.listenAddr(), "application/json", body) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if resp.Proto != "HTTP/2.0" { + t.Fatalf("expected HTTP/2.0, got %s", resp.Proto) + } + result, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(result), "jsonrpc") { + t.Fatalf("unexpected response: %s", result) + } +} + func apis() []rpc.API { return []rpc.API{ { diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 020b6fee27..bd87665125 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -64,7 +64,7 @@ function compile_fuzzer() { go get github.com/holiman/gofuzz-shim/testing if [[ $SANITIZER == *coverage* ]]; then - coverbuild $path $function $fuzzer $coverpkg + coverbuild $path $function $fuzzer else gofuzz-shim --func $function --package $package -f $file -o $fuzzer.a $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 08292a70ba..d6a30a17ca 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -326,7 +326,6 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error // Create header. head := c.makeHeader(toID, flagWhoareyou, 0) - head.AuthData = slices.Clone(c.buf.Bytes()) head.Nonce = packet.Nonce // Encode auth data. diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index d79677db55..b9570e561f 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -107,30 +107,30 @@ func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.I }) } // For IGDv1 and v1 services we should first try to add with extport. + var lastErr error for i := 0; i < retryCount+1; i++ { - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } // If above fails, we retry with a random port. // We retry several times because of possible port conflicts. - var err error for i := 0; i < randomCount; i++ { extport = n.randomPort() - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } - return 0, err + return 0, lastErr } func (n *upnp) randomPort() int { diff --git a/p2p/rlpx/rlpx.go b/p2p/rlpx/rlpx.go index a36ebc6b59..70658afdf8 100644 --- a/p2p/rlpx/rlpx.go +++ b/p2p/rlpx/rlpx.go @@ -24,6 +24,7 @@ import ( "crypto/ecdsa" "crypto/hmac" "crypto/rand" + "crypto/subtle" "encoding/binary" "errors" "fmt" @@ -34,7 +35,6 @@ import ( "time" "github.com/ethereum/go-ethereum/arbcrypto" - "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/rlp" @@ -677,6 +677,6 @@ func exportPubkey(pub *ecies.PublicKey) []byte { func xor(one, other []byte) (xor []byte) { xor = make([]byte, len(one)) - bitutil.XORBytes(xor, one, other) + subtle.XORBytes(xor, one, other) return xor } diff --git a/p2p/rlpx/rlpx_oracle_poc_test.go b/p2p/rlpx/rlpx_oracle_poc_test.go new file mode 100644 index 0000000000..3497f0026e --- /dev/null +++ b/p2p/rlpx/rlpx_oracle_poc_test.go @@ -0,0 +1,57 @@ +package rlpx + +import ( + "bytes" + "errors" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" +) + +func TestHandshakeECIESInvalidCurveOracle(t *testing.T) { + initKey, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + respKey, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + + init := handshakeState{ + initiator: true, + remote: ecies.ImportECDSAPublic(&respKey.PublicKey), + } + authMsg, err := init.makeAuthMsg(initKey) + if err != nil { + t.Fatal(err) + } + packet, err := init.sealEIP8(authMsg) + if err != nil { + t.Fatal(err) + } + + var recv handshakeState + if _, err := recv.readMsg(new(authMsgV4), respKey, bytes.NewReader(packet)); err != nil { + t.Fatalf("expected valid packet to decrypt: %v", err) + } + + tampered := append([]byte(nil), packet...) + if len(tampered) < 2+65 { + t.Fatalf("unexpected packet length %d", len(tampered)) + } + tampered[2] = 0x04 + for i := 1; i < 65; i++ { + tampered[2+i] = 0x00 + } + + var recv2 handshakeState + _, err = recv2.readMsg(new(authMsgV4), respKey, bytes.NewReader(tampered)) + if err == nil { + t.Fatal("expected decryption failure for invalid curve point") + } + if !errors.Is(err, ecies.ErrInvalidPublicKey) { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 99dd42514c..29b09cae8e 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -18,52 +18,63 @@ package tracker import ( "container/list" + "errors" "fmt" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" ) const ( - // trackedGaugeName is the prefix of the per-packet request tracking. trackedGaugeName = "p2p/tracked" - - // lostMeterName is the prefix of the per-packet request expirations. - lostMeterName = "p2p/lost" - - // staleMeterName is the prefix of the per-packet stale responses. - staleMeterName = "p2p/stale" - - // waitHistName is the prefix of the per-packet (req only) waiting time histograms. - waitHistName = "p2p/wait" + lostMeterName = "p2p/lost" + staleMeterName = "p2p/stale" + waitHistName = "p2p/wait" // maxTrackedPackets is a huge number to act as a failsafe on the number of // pending requests the node will track. It should never be hit unless an // attacker figures out a way to spin requests. - maxTrackedPackets = 100000 + maxTrackedPackets = 10000 ) -// request tracks sent network requests which have not yet received a response. -type request struct { - peer string - version uint // Protocol version +var ( + ErrNoMatchingRequest = errors.New("no matching request") + ErrTooManyItems = errors.New("response is larger than request allows") + ErrCollision = errors.New("request ID collision") + ErrCodeMismatch = errors.New("wrong response code for request") + ErrLimitReached = errors.New("request limit reached") + ErrStopped = errors.New("tracker stopped") +) - reqCode uint64 // Protocol message code of the request - resCode uint64 // Protocol message code of the expected response +// Request tracks sent network requests which have not yet received a response. +type Request struct { + ID uint64 // Request ID + Size int // Number/size of requested items + ReqCode uint64 // Protocol message code of the request + RespCode uint64 // Protocol message code of the expected response time time.Time // Timestamp when the request was made expire *list.Element // Expiration marker to untrack it } +type Response struct { + ID uint64 // Request ID of the response + MsgCode uint64 // Protocol message code + Size int // number/size of items in response +} + // Tracker is a pending network request tracker to measure how much time it takes // a remote peer to respond. type Tracker struct { - protocol string // Protocol capability identifier for the metrics - timeout time.Duration // Global timeout after which to drop a tracked packet + cap p2p.Cap // Protocol capability identifier for the metrics - pending map[uint64]*request // Currently pending requests + peer string // Peer ID + timeout time.Duration // Global timeout after which to drop a tracked packet + + pending map[uint64]*Request // Currently pending requests expire *list.List // Linked list tracking the expiration order wake *time.Timer // Timer tracking the expiration of the next item @@ -72,52 +83,53 @@ type Tracker struct { // New creates a new network request tracker to monitor how much time it takes to // fill certain requests and how individual peers perform. -func New(protocol string, timeout time.Duration) *Tracker { +func New(cap p2p.Cap, peerID string, timeout time.Duration) *Tracker { return &Tracker{ - protocol: protocol, - timeout: timeout, - pending: make(map[uint64]*request), - expire: list.New(), + cap: cap, + peer: peerID, + timeout: timeout, + pending: make(map[uint64]*Request), + expire: list.New(), } } // Track adds a network request to the tracker to wait for a response to arrive // or until the request it cancelled or times out. -func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint64, id uint64) { - if !metrics.Enabled() { - return - } +func (t *Tracker) Track(req Request) error { t.lock.Lock() defer t.lock.Unlock() + if t.expire == nil { + return ErrStopped + } + // If there's a duplicate request, we've just random-collided (or more probably, // we have a bug), report it. We could also add a metric, but we're not really // expecting ourselves to be buggy, so a noisy warning should be enough. - if _, ok := t.pending[id]; ok { - log.Error("Network request id collision", "protocol", t.protocol, "version", version, "code", reqCode, "id", id) - return + if _, ok := t.pending[req.ID]; ok { + log.Error("Network request id collision", "cap", t.cap, "code", req.ReqCode, "id", req.ID) + return ErrCollision } // If we have too many pending requests, bail out instead of leaking memory if pending := len(t.pending); pending >= maxTrackedPackets { - log.Error("Request tracker exceeded allowance", "pending", pending, "peer", peer, "protocol", t.protocol, "version", version, "code", reqCode) - return + log.Error("Request tracker exceeded allowance", "pending", pending, "peer", t.peer, "cap", t.cap, "code", req.ReqCode) + return ErrLimitReached } + // Id doesn't exist yet, start tracking it - t.pending[id] = &request{ - peer: peer, - version: version, - reqCode: reqCode, - resCode: resCode, - time: time.Now(), - expire: t.expire.PushBack(id), + req.time = time.Now() + req.expire = t.expire.PushBack(req.ID) + t.pending[req.ID] = &req + + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Inc(1) } - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, version, reqCode) - metrics.GetOrRegisterGauge(g, nil).Inc(1) // If we've just inserted the first item, start the expiration timer if t.wake == nil { t.wake = time.AfterFunc(t.timeout, t.clean) } + return nil } // clean is called automatically when a preset time passes without a response @@ -142,11 +154,10 @@ func (t *Tracker) clean() { t.expire.Remove(head) delete(t.pending, id) - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterGauge(g, nil).Dec(1) - - m := fmt.Sprintf("%s/%s/%d/%#02x", lostMeterName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterMeter(m, nil).Mark(1) + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Dec(1) + t.lostMeter(req.ReqCode).Mark(1) + } } t.schedule() } @@ -161,43 +172,92 @@ func (t *Tracker) schedule() { t.wake = time.AfterFunc(time.Until(t.pending[t.expire.Front().Value.(uint64)].time.Add(t.timeout)), t.clean) } -// Fulfil fills a pending request, if any is available, reporting on various metrics. -func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { - if !metrics.Enabled() { - return +// Stop reclaims resources of the tracker. +func (t *Tracker) Stop() { + t.lock.Lock() + defer t.lock.Unlock() + + if t.wake != nil { + t.wake.Stop() + t.wake = nil + } + if metrics.Enabled() { + // Ensure metrics are decremented for pending requests. + counts := make(map[uint64]int64) + for _, req := range t.pending { + counts[req.ReqCode]++ + } + for code, count := range counts { + t.trackedGauge(code).Dec(count) + } } + clear(t.pending) + t.expire = nil +} + +// Fulfil fills a pending request, if any is available, reporting on various metrics. +func (t *Tracker) Fulfil(resp Response) error { t.lock.Lock() defer t.lock.Unlock() // If it's a non existing request, track as stale response - req, ok := t.pending[id] + req, ok := t.pending[resp.ID] if !ok { - m := fmt.Sprintf("%s/%s/%d/%#02x", staleMeterName, t.protocol, version, code) - metrics.GetOrRegisterMeter(m, nil).Mark(1) - return + if metrics.Enabled() { + t.staleMeter(resp.MsgCode).Mark(1) + } + return ErrNoMatchingRequest } + // If the response is funky, it might be some active attack - if req.peer != peer || req.version != version || req.resCode != code { - log.Warn("Network response id collision", - "have", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, version, code), - "want", fmt.Sprintf("%s:%s/%d:%d", peer, t.protocol, req.version, req.resCode), + if req.RespCode != resp.MsgCode { + log.Warn("Network response code collision", + "have", fmt.Sprintf("%s:%s/%d:%d", t.peer, t.cap.Name, t.cap.Version, resp.MsgCode), + "want", fmt.Sprintf("%s:%s/%d:%d", t.peer, t.cap.Name, t.cap.Version, req.RespCode), ) - return + return ErrCodeMismatch } + if resp.Size > req.Size { + return ErrTooManyItems + } + // Everything matches, mark the request serviced and meter it + wasHead := req.expire.Prev() == nil t.expire.Remove(req.expire) - delete(t.pending, id) - if req.expire.Prev() == nil { + delete(t.pending, req.ID) + if wasHead { if t.wake.Stop() { t.schedule() } } - g := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.protocol, req.version, req.reqCode) - metrics.GetOrRegisterGauge(g, nil).Dec(1) - h := fmt.Sprintf("%s/%s/%d/%#02x", waitHistName, t.protocol, req.version, req.reqCode) + // Update request metrics. + if metrics.Enabled() { + t.trackedGauge(req.ReqCode).Dec(1) + t.waitHistogram(req.ReqCode).Update(time.Since(req.time).Microseconds()) + } + return nil +} + +func (t *Tracker) trackedGauge(code uint64) *metrics.Gauge { + name := fmt.Sprintf("%s/%s/%d/%#02x", trackedGaugeName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterGauge(name, nil) +} + +func (t *Tracker) lostMeter(code uint64) *metrics.Meter { + name := fmt.Sprintf("%s/%s/%d/%#02x", lostMeterName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterMeter(name, nil) +} + +func (t *Tracker) staleMeter(code uint64) *metrics.Meter { + name := fmt.Sprintf("%s/%s/%d/%#02x", staleMeterName, t.cap.Name, t.cap.Version, code) + return metrics.GetOrRegisterMeter(name, nil) +} + +func (t *Tracker) waitHistogram(code uint64) metrics.Histogram { + name := fmt.Sprintf("%s/%s/%d/%#02x", waitHistName, t.cap.Name, t.cap.Version, code) sampler := func() metrics.Sample { return metrics.NewBoundedHistogramSample() } - metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(req.time).Microseconds()) + return metrics.GetOrRegisterHistogramLazy(name, nil, sampler) } diff --git a/p2p/tracker/tracker_test.go b/p2p/tracker/tracker_test.go new file mode 100644 index 0000000000..a37a59f70f --- /dev/null +++ b/p2p/tracker/tracker_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracker + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" +) + +// This checks that metrics gauges for pending requests are be decremented when a +// Tracker is stopped. +func TestMetricsOnStop(t *testing.T) { + metrics.Enable() + + cap := p2p.Cap{Name: "test", Version: 1} + tr := New(cap, "peer1", time.Minute) + + // Track some requests with different ReqCodes. + var id uint64 + for i := 0; i < 3; i++ { + tr.Track(Request{ID: id, ReqCode: 0x01, RespCode: 0x02, Size: 1}) + id++ + } + for i := 0; i < 5; i++ { + tr.Track(Request{ID: id, ReqCode: 0x03, RespCode: 0x04, Size: 1}) + id++ + } + + gauge1 := tr.trackedGauge(0x01) + gauge2 := tr.trackedGauge(0x03) + + if gauge1.Snapshot().Value() != 3 { + t.Fatalf("gauge1 value mismatch: got %d, want 3", gauge1.Snapshot().Value()) + } + if gauge2.Snapshot().Value() != 5 { + t.Fatalf("gauge2 value mismatch: got %d, want 5", gauge2.Snapshot().Value()) + } + + tr.Stop() + + if gauge1.Snapshot().Value() != 0 { + t.Fatalf("gauge1 value after stop: got %d, want 0", gauge1.Snapshot().Value()) + } + if gauge2.Snapshot().Value() != 0 { + t.Fatalf("gauge2 value after stop: got %d, want 0", gauge2.Snapshot().Value()) + } +} diff --git a/params/config.go b/params/config.go index 794e4e62b1..95aaf2a710 100644 --- a/params/config.go +++ b/params/config.go @@ -1436,7 +1436,6 @@ func (err *ConfigCompatError) Error() string { // phases. type Rules struct { IsArbitrum, IsStylus, IsDia bool - ChainID *big.Int ArbOSVersion uint64 IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsEIP2929, IsEIP4762 bool @@ -1448,10 +1447,6 @@ type Rules struct { // Rules ensures c's ChainID is not nil. func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64, currentArbosVersion uint64) Rules { - chainID := c.ChainID - if chainID == nil { - chainID = new(big.Int) - } // disallow setting Merge out of order isMerge = isMerge && c.IsLondon(num) isVerkle := isMerge && c.IsVerkle(num, timestamp) @@ -1459,7 +1454,6 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64, curren IsArbitrum: c.IsArbitrum(), IsStylus: c.IsArbitrum() && currentArbosVersion >= ArbosVersion_Stylus, IsDia: c.IsArbitrum() && currentArbosVersion >= ArbosVersion_Dia, - ChainID: new(big.Int).Set(chainID), ArbOSVersion: currentArbosVersion, IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), diff --git a/params/protocol_params.go b/params/protocol_params.go index fe35011517..0229ce19dd 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -32,7 +32,7 @@ const ( MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. ExpByteGas uint64 = 10 // Times ceil(log256(exponent)) for the EXP instruction. - SloadGas uint64 = 50 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. + SloadGas uint64 = 50 // CallValueTransferGas uint64 = 9000 // Paid for CALL when the value transfer is non-zero. CallNewAccountGas uint64 = 25000 // Paid for CALL when the destination address didn't exist prior. TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. @@ -82,7 +82,7 @@ const ( CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. ExpGas uint64 = 10 // Once per EXP instruction LogGas uint64 = 375 // Per LOG* operation. - CopyGas uint64 = 3 // + CopyGas uint64 = 3 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. StackLimit uint64 = 1024 // Maximum size of VM stack allowed. TierStepGas uint64 = 0 // Once per operation, for a selection of them. LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. diff --git a/rlp/encode.go b/rlp/encode.go index ed99275739..9d04e6324a 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -102,6 +102,29 @@ func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { return buf.size(), &encReader{buf: buf}, nil } +// EncodeToRawList encodes val as an RLP list and returns it as a RawList. +func EncodeToRawList[T any](val []T) (RawList[T], error) { + if len(val) == 0 { + return RawList[T]{}, nil + } + + // Encode the value to an internal buffer. + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { + return RawList[T]{}, err + } + + // Create the RawList. RawList assumes the initial list header is padded + // 9 bytes, so we have to determine the offset where the value should be + // placed. + contentSize := buf.lheads[0].size + bytes := make([]byte, contentSize+9) + offset := 9 - headsize(uint64(contentSize)) + buf.copyTo(bytes[offset:]) + return RawList[T]{enc: bytes, length: len(val)}, nil +} + type listhead struct { offset int // index of this header in string data size int // total size of encoded data (including list headers) diff --git a/rlp/iterator.go b/rlp/iterator.go index 95bd3f2582..9e41cec947 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -16,44 +16,74 @@ package rlp -type listIterator struct { - data []byte - next []byte - err error +// Iterator is an iterator over the elements of an encoded container. +type Iterator struct { + data []byte + next []byte + offset int + err error } -// NewListIterator creates an iterator for the (list) represented by data -func NewListIterator(data RawValue) (*listIterator, error) { +// NewListIterator creates an iterator for the (list) represented by data. +func NewListIterator(data RawValue) (Iterator, error) { k, t, c, err := readKind(data) if err != nil { - return nil, err + return Iterator{}, err } if k != List { - return nil, ErrExpectedList - } - it := &listIterator{ - data: data[t : t+c], + return Iterator{}, ErrExpectedList } + it := newIterator(data[t:t+c], int(t)) return it, nil } -// Next forwards the iterator one step, returns true if it was not at end yet -func (it *listIterator) Next() bool { +func newIterator(data []byte, initialOffset int) Iterator { + return Iterator{data: data, offset: initialOffset} +} + +// Next forwards the iterator one step. +// Returns true if there is a next item or an error occurred on this step (check Err()). +// On parse error, the iterator is marked finished and subsequent calls return false. +func (it *Iterator) Next() bool { if len(it.data) == 0 { return false } _, t, c, err := readKind(it.data) - it.next = it.data[:t+c] - it.data = it.data[t+c:] - it.err = err + if err != nil { + it.next = nil + it.err = err + // Mark iteration as finished to avoid potential infinite loops on subsequent Next calls. + it.data = nil + return true + } + length := t + c + it.next = it.data[:length] + it.data = it.data[length:] + it.offset += int(length) + it.err = nil return true } -// Value returns the current value -func (it *listIterator) Value() []byte { +// Value returns the current value. +func (it *Iterator) Value() []byte { return it.next } -func (it *listIterator) Err() error { +// Count returns the remaining number of items. +// Note this is O(n) and the result may be incorrect if the list data is invalid. +// The returned count is always an upper bound on the remaining items +// that will be visited by the iterator. +func (it *Iterator) Count() int { + count, _ := CountValues(it.data) + return count +} + +// Offset returns the offset of the current value into the list data. +func (it *Iterator) Offset() int { + return it.offset - len(it.next) +} + +// Err returns the error that caused Next to return false, if any. +func (it *Iterator) Err() error { return it.err } diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go index a22aaec862..275d4371c7 100644 --- a/rlp/iterator_test.go +++ b/rlp/iterator_test.go @@ -17,6 +17,7 @@ package rlp import ( + "io" "testing" "github.com/ethereum/go-ethereum/common/hexutil" @@ -38,14 +39,25 @@ func TestIterator(t *testing.T) { t.Fatal("expected two elems, got zero") } txs := it.Value() + if offset := it.Offset(); offset != 3 { + t.Fatal("wrong offset", offset, "want 3") + } + // Check that uncles exist if !it.Next() { t.Fatal("expected two elems, got one") } + if offset := it.Offset(); offset != 219 { + t.Fatal("wrong offset", offset, "want 219") + } + txit, err := NewListIterator(txs) if err != nil { t.Fatal(err) } + if c := txit.Count(); c != 2 { + t.Fatal("wrong Count:", c) + } var i = 0 for txit.Next() { if txit.err != nil { @@ -57,3 +69,65 @@ func TestIterator(t *testing.T) { t.Errorf("count wrong, expected %d got %d", i, exp) } } + +func TestIteratorErrors(t *testing.T) { + tests := []struct { + input []byte + wantCount int // expected Count before iterating + wantErr error + }{ + // Second item string header claims 3 bytes content, but only 2 remain. + {unhex("C4 01 83AABB"), 2, ErrValueTooLarge}, + // Second item truncated: B9 requires 2 size bytes, none available. + {unhex("C2 01 B9"), 2, io.ErrUnexpectedEOF}, + // 0x05 should be encoded directly, not as 81 05. + {unhex("C3 01 8105"), 2, ErrCanonSize}, + // Long-form string header B8 used for 1-byte content (< 56). + {unhex("C4 01 B801AA"), 2, ErrCanonSize}, + // Long-form list header F8 used for 1-byte content (< 56). + {unhex("C4 01 F80101"), 2, ErrCanonSize}, + } + for _, tt := range tests { + it, err := NewListIterator(tt.input) + if err != nil { + t.Fatal("NewListIterator error:", err) + } + if c := it.Count(); c != tt.wantCount { + t.Fatalf("%x: Count = %d, want %d", tt.input, c, tt.wantCount) + } + n := 0 + for it.Next() { + if it.Err() != nil { + break + } + n++ + } + if wantN := tt.wantCount - 1; n != wantN { + t.Fatalf("%x: got %d valid items, want %d", tt.input, n, wantN) + } + if it.Err() != tt.wantErr { + t.Fatalf("%x: got error %v, want %v", tt.input, it.Err(), tt.wantErr) + } + if it.Next() { + t.Fatalf("%x: Next returned true after error", tt.input) + } + } +} + +func FuzzIteratorCount(f *testing.F) { + examples := [][]byte{unhex("010203"), unhex("018142"), unhex("01830202")} + for _, e := range examples { + f.Add(e) + } + f.Fuzz(func(t *testing.T, in []byte) { + it := newIterator(in, 0) + count := it.Count() + i := 0 + for it.Next() { + i++ + } + if i != count { + t.Fatalf("%x: count %d not equal to %d iterations", in, count, i) + } + }) +} diff --git a/rlp/raw.go b/rlp/raw.go index cec90346a1..08ec667158 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -17,8 +17,10 @@ package rlp import ( + "fmt" "io" "reflect" + "slices" ) // RawValue represents an encoded RLP value and can be used to delay @@ -28,6 +30,144 @@ type RawValue []byte var rawValueType = reflect.TypeFor[RawValue]() +// RawList represents an encoded RLP list. +type RawList[T any] struct { + // The list is stored in encoded form. + // Note this buffer has some special properties: + // + // - if the buffer is nil, it's the zero value, representing + // an empty list. + // - if the buffer is non-nil, it must have a length of at least + // 9 bytes, which is reserved padding for the encoded list header. + // The remaining bytes, enc[9:], store the content bytes of the list. + // + // The implementation code mostly works with the Content method because it + // returns something valid either way. + enc []byte + + // length holds the number of items in the list. + length int +} + +// Content returns the RLP-encoded data of the list. +// This does not include the list-header. +// The return value is a direct reference to the internal buffer, not a copy. +func (r *RawList[T]) Content() []byte { + if r.enc == nil { + return nil + } else { + return r.enc[9:] + } +} + +// EncodeRLP writes the encoded list to the writer. +func (r RawList[T]) EncodeRLP(w io.Writer) error { + _, err := w.Write(r.Bytes()) + return err +} + +// Bytes returns the RLP encoding of the list. +// Note the return value aliases the internal buffer. +func (r *RawList[T]) Bytes() []byte { + if r == nil || r.enc == nil { + return []byte{0xC0} // zero value encodes as empty list + } + n := puthead(r.enc, 0xC0, 0xF7, uint64(len(r.Content()))) + copy(r.enc[9-n:], r.enc[:n]) + return r.enc[9-n:] +} + +// DecodeRLP decodes the list. This does not perform validation of the items! +func (r *RawList[T]) DecodeRLP(s *Stream) error { + k, size, err := s.Kind() + if err != nil { + return err + } + if k != List { + return fmt.Errorf("%w for %T", ErrExpectedList, r) + } + enc := make([]byte, 9+size) + if err := s.readFull(enc[9:]); err != nil { + return err + } + n, err := CountValues(enc[9:]) + if err != nil { + if err == ErrValueTooLarge { + return ErrElemTooLarge + } + return err + } + *r = RawList[T]{enc: enc, length: n} + return nil +} + +// Items decodes and returns all items in the list. +func (r *RawList[T]) Items() ([]T, error) { + items := make([]T, r.Len()) + it := r.ContentIterator() + for i := 0; it.Next(); i++ { + if err := DecodeBytes(it.Value(), &items[i]); err != nil { + return items[:i], err + } + } + return items, nil +} + +// Len returns the number of items in the list. +func (r *RawList[T]) Len() int { + return r.length +} + +// Size returns the encoded size of the list. +func (r *RawList[T]) Size() uint64 { + return ListSize(uint64(len(r.Content()))) +} + +// ContentIterator returns an iterator over the content of the list. +// Note the offsets returned by iterator.Offset are relative to the +// Content bytes of the list. +func (r *RawList[T]) ContentIterator() Iterator { + return newIterator(r.Content(), 0) +} + +// Append adds an item to the end of the list. +func (r *RawList[T]) Append(item T) error { + if r.enc == nil { + r.enc = make([]byte, 9) + } + + eb := getEncBuffer() + defer encBufferPool.Put(eb) + + if err := eb.encode(item); err != nil { + return err + } + prevEnd := len(r.enc) + end := prevEnd + eb.size() + r.enc = slices.Grow(r.enc, eb.size())[:end] + eb.copyTo(r.enc[prevEnd:end]) + r.length++ + return nil +} + +// AppendRaw adds an encoded item to the list. +// The given byte slice must contain exactly one RLP value. +func (r *RawList[T]) AppendRaw(b []byte) error { + _, tagsize, contentsize, err := readKind(b) + if err != nil { + return err + } + if tagsize+contentsize != uint64(len(b)) { + return fmt.Errorf("rlp: input has trailing bytes in AppendRaw") + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, b...) + r.length++ + return nil +} + // StringSize returns the encoded size of a string. func StringSize(s string) uint64 { switch n := len(s); n { @@ -145,13 +285,52 @@ func CountValues(b []byte) (int, error) { for ; len(b) > 0; i++ { _, tagsize, size, err := readKind(b) if err != nil { - return 0, err + return i + 1, err } b = b[tagsize+size:] } return i, nil } +// SplitListValues extracts the raw elements from the list RLP-encoding blob. +// +// Note: the returned slice must not be modified, as it shares the same +// backing array as the original slice. It's acceptable to deep-copy the elements +// out if necessary, but let's stick with this approach for less allocation +// overhead. +func SplitListValues(b []byte) ([][]byte, error) { + b, _, err := SplitList(b) + if err != nil { + return nil, err + } + n, err := CountValues(b) + if err != nil { + return nil, err + } + var elements = make([][]byte, 0, n) + + for len(b) > 0 { + _, tagsize, size, err := readKind(b) + if err != nil { + return nil, err + } + elements = append(elements, b[:tagsize+size]) + b = b[tagsize+size:] + } + return elements, nil +} + +// MergeListValues takes a list of raw elements and rlp-encodes them as list. +func MergeListValues(elems [][]byte) ([]byte, error) { + w := NewEncoderBuffer(nil) + offset := w.List() + for _, elem := range elems { + w.Write(elem) + } + w.ListEnd(offset) + return w.ToBytes(), nil +} + func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { if len(buf) == 0 { return 0, 0, 0, io.ErrUnexpectedEOF diff --git a/rlp/raw_test.go b/rlp/raw_test.go index 7b3255eca3..112c5d7897 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -19,11 +19,259 @@ package rlp import ( "bytes" "errors" + "fmt" "io" + "reflect" "testing" "testing/quick" ) +type rawListTest[T any] struct { + input string + content string + items []T + length int +} + +func (test rawListTest[T]) name() string { + return fmt.Sprintf("%T-%d", *new(T), test.length) +} + +func (test rawListTest[T]) run(t *testing.T) { + // check decoding and properties + input := unhex(test.input) + inputSize := len(input) + var rl RawList[T] + if err := DecodeBytes(input, &rl); err != nil { + t.Fatal("decode failed:", err) + } + if l := rl.Len(); l != test.length { + t.Fatalf("wrong Len %d, want %d", l, test.length) + } + if sz := rl.Size(); sz != uint64(inputSize) { + t.Fatalf("wrong Size %d, want %d", sz, inputSize) + } + items, err := rl.Items() + if err != nil { + t.Fatal("Items failed:", err) + } + if !reflect.DeepEqual(items, test.items) { + t.Fatal("wrong items:", items) + } + if !bytes.Equal(rl.Content(), unhex(test.content)) { + t.Fatalf("wrong Content %x, want %s", rl.Content(), test.content) + } + if !bytes.Equal(rl.Bytes(), unhex(test.input)) { + t.Fatalf("wrong Bytes %x, want %s", rl.Bytes(), test.input) + } + + // check iterator + it := rl.ContentIterator() + i := 0 + for it.Next() { + var item T + if err := DecodeBytes(it.Value(), &item); err != nil { + t.Fatalf("item %d decode error: %v", i, err) + } + if !reflect.DeepEqual(item, items[i]) { + t.Fatalf("iterator has wrong item %v at %d", item, i) + } + i++ + } + if i != test.length { + t.Fatalf("iterator produced %d values, want %d", i, test.length) + } + if it.Err() != nil { + t.Fatalf("iterator error: %v", it.Err()) + } + + // check encoding round trip + output, err := EncodeToBytes(&rl) + if err != nil { + t.Fatal("encode error:", err) + } + if !bytes.Equal(output, unhex(test.input)) { + t.Fatalf("encoding does not round trip: %x", output) + } + + // check EncodeToRawList on items produces same bytes + encRL, err := EncodeToRawList(test.items) + if err != nil { + t.Fatal("EncodeToRawList error:", err) + } + encRLOutput, err := EncodeToBytes(&encRL) + if err != nil { + t.Fatal("EncodeToBytes of encoded list failed:", err) + } + if !bytes.Equal(encRLOutput, output) { + t.Fatalf("wrong encoding of EncodeToRawList result: %x", encRLOutput) + } +} + +func TestRawList(t *testing.T) { + tests := []interface { + name() string + run(t *testing.T) + }{ + rawListTest[uint64]{ + input: "C0", + content: "", + items: []uint64{}, + length: 0, + }, + rawListTest[uint64]{ + input: "C3010203", + content: "010203", + items: []uint64{1, 2, 3}, + length: 3, + }, + rawListTest[simplestruct]{ + input: "C6C20102C20304", + content: "C20102C20304", + items: []simplestruct{{1, "\x02"}, {3, "\x04"}}, + length: 2, + }, + rawListTest[string]{ + input: "F83C836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + content: "836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + items: []string{"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", "lll", "mmm", "nnn", "ooo"}, + length: 15, + }, + } + + for _, test := range tests { + t.Run(test.name(), test.run) + } +} + +func TestRawListEmpty(t *testing.T) { + // zero value list + var rl RawList[uint64] + b, _ := EncodeToBytes(&rl) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("empty RawList has wrong encoding %x", b) + } + if rl.Len() != 0 { + t.Fatalf("empty list has Len %d", rl.Len()) + } + if rl.Size() != 1 { + t.Fatalf("empty list has Size %d", rl.Size()) + } + if len(rl.Content()) > 0 { + t.Fatalf("empty list has non-empty Content") + } + if !bytes.Equal(rl.Bytes(), []byte{0xC0}) { + t.Fatalf("empty list has wrong encoding") + } + + // nil pointer + var nilptr *RawList[uint64] + b, _ = EncodeToBytes(nilptr) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } +} + +// This checks that *RawList works in an 'optional' context. +func TestRawListOptional(t *testing.T) { + type foo struct { + L *RawList[uint64] `rlp:"optional"` + } + // nil pointer encoding + var empty foo + b, _ := EncodeToBytes(empty) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } + // decoding + var dec foo + if err := DecodeBytes(unhex("C0"), &dec); err != nil { + t.Fatal(err) + } + if dec.L != nil { + t.Fatal("rawlist was decoded as non-nil") + } +} + +func TestRawListAppend(t *testing.T) { + var rl RawList[simplestruct] + + v1 := simplestruct{1, "one"} + v2 := simplestruct{2, "two"} + if err := rl.Append(v1); err != nil { + t.Fatal("append 1 failed:", err) + } + if err := rl.Append(v2); err != nil { + t.Fatal("append 2 failed:", err) + } + + if rl.Len() != 2 { + t.Fatalf("wrong Len %d", rl.Len()) + } + if rl.Size() != 13 { + t.Fatalf("wrong Size %d", rl.Size()) + } + if !bytes.Equal(rl.Content(), unhex("C501836F6E65 C5028374776F")) { + t.Fatalf("wrong Content %x", rl.Content()) + } + encoded, _ := EncodeToBytes(&rl) + if !bytes.Equal(encoded, unhex("CC C501836F6E65 C5028374776F")) { + t.Fatalf("wrong encoding %x", encoded) + } +} + +func TestRawListAppendRaw(t *testing.T) { + var rl RawList[uint64] + + if err := rl.AppendRaw(unhex("01")); err != nil { + t.Fatal("AppendRaw(01) failed:", err) + } + if err := rl.AppendRaw(unhex("820102")); err != nil { + t.Fatal("AppendRaw(820102) failed:", err) + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after valid appends", rl.Len()) + } + + if err := rl.AppendRaw(nil); err == nil { + t.Fatal("AppendRaw(nil) should fail") + } + if err := rl.AppendRaw(unhex("0102")); err == nil { + t.Fatal("AppendRaw(0102) should fail due to trailing bytes") + } + if err := rl.AppendRaw(unhex("8201")); err == nil { + t.Fatal("AppendRaw(8201) should fail due to truncated value") + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after invalid appends, want 2", rl.Len()) + } +} + +func TestRawListDecodeInvalid(t *testing.T) { + tests := []struct { + input string + err error + }{ + // Single item with non-canonical size (0x81 wrapping byte <= 0x7F). + {input: "C28142", err: ErrCanonSize}, + // Single item claiming more bytes than available in the list. + {input: "C484020202", err: ErrElemTooLarge}, + // Two items, second has non-canonical size. + {input: "C3018142", err: ErrCanonSize}, + // Two items, second claims more bytes than remain in the list. + {input: "C401830202", err: ErrElemTooLarge}, + // Item is a sub-list whose declared size exceeds available bytes. + {input: "C3C40102", err: ErrElemTooLarge}, + } + for _, test := range tests { + var rl RawList[RawValue] + err := DecodeBytes(unhex(test.input), &rl) + if !errors.Is(err, test.err) { + t.Errorf("input %s: error mismatch: got %v, want %v", test.input, err, test.err) + } + } +} + func TestCountValues(t *testing.T) { tests := []struct { input string // note: spaces in input are stripped by unhex @@ -40,9 +288,9 @@ func TestCountValues(t *testing.T) { {"820101 820202 8403030303 04", 4, nil}, // size errors - {"8142", 0, ErrCanonSize}, - {"01 01 8142", 0, ErrCanonSize}, - {"02 84020202", 0, ErrValueTooLarge}, + {"8142", 1, ErrCanonSize}, + {"01 01 8142", 3, ErrCanonSize}, + {"02 84020202", 2, ErrValueTooLarge}, { input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", @@ -336,3 +584,269 @@ func TestBytesSize(t *testing.T) { } } } + +func TestSplitListValues(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + want []string // hex-encoded expected elements + wantErr error + }{ + { + name: "empty list", + input: "C0", + want: []string{}, + }, + { + name: "single byte element", + input: "C101", + want: []string{"01"}, + }, + { + name: "single empty string", + input: "C180", + want: []string{"80"}, + }, + { + name: "two byte elements", + input: "C20102", + want: []string{"01", "02"}, + }, + { + name: "three elements", + input: "C3010203", + want: []string{"01", "02", "03"}, + }, + { + name: "mixed size elements", + input: "C80182020283030303", + want: []string{"01", "820202", "83030303"}, + }, + { + name: "string elements", + input: "C88363617483646F67", + want: []string{"83636174", "83646F67"}, // cat,dog + }, + { + name: "nested list element", + input: "C4C3010203", // [[1,2,3]] + want: []string{"C3010203"}, // [1,2,3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + want: []string{"C20102", "C20304"}, // [1,2], [3,4] + }, + { + name: "large list", + input: "C6010203040506", + want: []string{"01", "02", "03", "04", "05", "06"}, + }, + { + name: "list with empty strings", + input: "C3808080", + want: []string{"80", "80", "80"}, + }, + // Error cases + { + name: "single byte", + input: "01", + wantErr: ErrExpectedList, + }, + { + name: "string", + input: "83636174", + wantErr: ErrExpectedList, + }, + { + name: "empty input", + input: "", + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "invalid list - value too large", + input: "C60102030405", + wantErr: ErrValueTooLarge, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SplitListValues(unhex(tt.input)) + if !errors.Is(err, tt.wantErr) { + t.Errorf("SplitListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + if len(got) != len(tt.want) { + t.Errorf("SplitListValues() got %d elements, want %d", len(got), len(tt.want)) + return + } + for i, elem := range got { + want := unhex(tt.want[i]) + if !bytes.Equal(elem, want) { + t.Errorf("SplitListValues() element[%d] = %x, want %x", i, elem, want) + } + } + }) + } +} + +func TestMergeListValues(t *testing.T) { + tests := []struct { + name string + elems []string // hex-encoded RLP elements + want string // hex-encoded expected result + wantErr error + }{ + { + name: "empty list", + elems: []string{}, + want: "C0", + }, + { + name: "single byte element", + elems: []string{"01"}, + want: "C101", + }, + { + name: "single empty string", + elems: []string{"80"}, + want: "C180", + }, + { + name: "two byte elements", + elems: []string{"01", "02"}, + want: "C20102", + }, + { + name: "three elements", + elems: []string{"01", "02", "03"}, + want: "C3010203", + }, + { + name: "mixed size elements", + elems: []string{"01", "820202", "83030303"}, + want: "C80182020283030303", + }, + { + name: "string elements", + elems: []string{"83636174", "83646F67"}, // cat, dog + want: "C88363617483646F67", + }, + { + name: "nested list element", + elems: []string{"C20102", "03"}, // [[1, 2], 3] + want: "C4C2010203", + }, + { + name: "multiple nested lists", + elems: []string{"C20102", "C3030405"}, // [[1,2],[3,4,5]], + want: "C7C20102C3030405", + }, + { + name: "large list", + elems: []string{"01", "02", "03", "04", "05", "06"}, + want: "C6010203040506", + }, + { + name: "list with empty strings", + elems: []string{"80", "80", "80"}, + want: "C3808080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + elems := make([][]byte, len(tt.elems)) + for i, s := range tt.elems { + elems[i] = unhex(s) + } + got, err := MergeListValues(elems) + if !errors.Is(err, tt.wantErr) { + t.Errorf("MergeListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + want := unhex(tt.want) + if !bytes.Equal(got, want) { + t.Errorf("MergeListValues() = %x, want %x", got, want) + } + }) + } +} + +func TestSplitMergeList(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + }{ + { + name: "empty list", + input: "C0", + }, + { + name: "single byte element", + input: "C101", + }, + { + name: "two byte elements", + input: "C20102", + }, + { + name: "three elements", + input: "C3010203", + }, + { + name: "mixed size elements", + input: "C80182020283030303", + }, + { + name: "string elements", + input: "C88363617483646F67", // [cat, dog] + }, + { + name: "nested list element", + input: "C4C2010203", // [[1,2],3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + }, + { + name: "large list", + input: "C6010203040506", // [1,2,3,4,5,6] + }, + { + name: "list with empty strings", + input: "C3808080", // ["", "", ""] + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original := unhex(tt.input) + + // Split the list + elements, err := SplitListValues(original) + if err != nil { + t.Fatalf("SplitListValues() error = %v", err) + } + + // Merge back + merged, err := MergeListValues(elements) + if err != nil { + t.Fatalf("MergeListValues() error = %v", err) + } + + // The merged result should match the original + if !bytes.Equal(merged, original) { + t.Errorf("Round trip failed: original = %x, merged = %x", original, merged) + } + }) + } +} diff --git a/rpc/client.go b/rpc/client.go index b0ed124797..fb1ab006fa 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -126,7 +126,7 @@ func (c *Client) newClientConn(conn ServerCodec) *clientConn { ctx := context.Background() ctx = context.WithValue(ctx, clientContextKey{}, c) ctx = context.WithValue(ctx, peerInfoContextKey{}, conn.peerInfo()) - handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize) + handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize, nil) return &clientConn{conn, handler} } diff --git a/rpc/handler.go b/rpc/handler.go index 06cc9c28c4..b8342f7ed6 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -28,7 +28,10 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) // handler handles JSON-RPC messages. There is one handler per connection. Note that @@ -65,6 +68,7 @@ type handler struct { allowSubscribe bool batchRequestLimit int batchResponseMaxSize int + tracerProvider trace.TracerProvider subLock sync.Mutex serverSubs map[ID]*Subscription @@ -73,9 +77,10 @@ type handler struct { type callProc struct { ctx context.Context notifiers []*Notifier + isBatch bool } -func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int) *handler { +func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int, tracerProvider trace.TracerProvider) *handler { rootCtx, cancelRoot := context.WithCancel(connCtx) h := &handler{ reg: reg, @@ -90,6 +95,7 @@ func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg * log: log.Root(), batchRequestLimit: batchRequestLimit, batchResponseMaxSize: batchResponseMaxSize, + tracerProvider: tracerProvider, } if conn.remoteAddr() != "" { h.log = h.log.New("conn", conn.remoteAddr()) @@ -197,6 +203,7 @@ func (h *handler) handleBatch(msgs []*jsonrpcMessage) { // Process calls on a goroutine because they may block indefinitely: h.startCallProc(func(cp *callProc) { + cp.isBatch = true var ( timer *time.Timer cancel context.CancelFunc @@ -497,40 +504,65 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage if msg.isSubscribe() { return h.handleSubscribe(cp, msg) } - var callb *callback if msg.isUnsubscribe() { - callb = h.unsubscribeCb - } else { - // Check method name length - if len(msg.Method) > maxMethodNameLength { - return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)}) + args, err := parsePositionalArguments(msg.Params, h.unsubscribeCb.argTypes) + if err != nil { + return msg.errorResponse(&invalidParamsError{err.Error()}) } - callb = h.reg.callback(msg.Method) + return h.runMethod(cp.ctx, msg, h.unsubscribeCb, args) + } + + // Check method name length + if len(msg.Method) > maxMethodNameLength { + return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)}) } + callb, service, method := h.reg.callback(msg.Method) + + // If the method is not found, return an error. if callb == nil { return msg.errorResponse(&methodNotFoundError{method: msg.Method}) } - args, err := parsePositionalArguments(msg.Params, callb.argTypes) - if err != nil { - return msg.errorResponse(&invalidParamsError{err.Error()}) + // Start root span for the request. + rpcInfo := telemetry.RPCInfo{ + System: "jsonrpc", + Service: service, + Method: method, + RequestID: string(msg.ID), + } + attrib := []telemetry.Attribute{ + telemetry.BoolAttribute("rpc.batch", cp.isBatch), + } + ctx, spanEnd := telemetry.StartServerSpan(cp.ctx, h.tracer(), rpcInfo, attrib...) + defer spanEnd(nil) // don't propagate errors to parent spans + + // Start tracing span before parsing arguments. + _, _, pSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.parsePositionalArguments") + args, pErr := parsePositionalArguments(msg.Params, callb.argTypes) + pSpanEnd(&pErr) + if pErr != nil { + return msg.errorResponse(&invalidParamsError{pErr.Error()}) } start := time.Now() - answer := h.runMethod(cp.ctx, msg, callb, args) - // Collect the statistics for RPC calls if metrics is enabled. - // We only care about pure rpc call. Filter out subscription. - if callb != h.unsubscribeCb { - rpcRequestGauge.Inc(1) - if answer.Error != nil { - failedRequestGauge.Inc(1) - } else { - successfulRequestGauge.Inc(1) - } - rpcServingTimer.UpdateSince(start) - updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) + // Start tracing span before running the method. + rctx, _, rSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.runMethod") + answer := h.runMethod(rctx, msg, callb, args) + var rErr error + if answer.Error != nil { + rErr = errors.New(answer.Error.Message) } + rSpanEnd(&rErr) + // Collect the statistics for RPC calls if metrics is enabled. + rpcRequestGauge.Inc(1) + if answer.Error != nil { + failedRequestGauge.Inc(1) + } else { + successfulRequestGauge.Inc(1) + } + rpcServingTimer.UpdateSince(start) + updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) return answer } @@ -568,17 +600,33 @@ func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMes n := &Notifier{h: h, namespace: namespace} cp.notifiers = append(cp.notifiers, n) ctx := context.WithValue(cp.ctx, notifierKey{}, n) - return h.runMethod(ctx, msg, callb, args) } +// tracer returns the OpenTelemetry Tracer for RPC call tracing. +func (h *handler) tracer() trace.Tracer { + if h.tracerProvider == nil { + // Default to global TracerProvider if none is set. + // Note: If no TracerProvider is set, the default is a no-op TracerProvider. + // See https://pkg.go.dev/go.opentelemetry.io/otel#GetTracerProvider + return otel.Tracer("") + } + return h.tracerProvider.Tracer("") +} + // runMethod runs the Go callback for an RPC method. -func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value) *jsonrpcMessage { +func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value, attributes ...telemetry.Attribute) *jsonrpcMessage { result, err := callb.call(ctx, msg.Method, args) if err != nil { return msg.errorResponse(err) } - return msg.response(result) + _, _, spanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.encodeJSONResponse", attributes...) + response := msg.response(result) + if response.Error != nil { + err = errors.New(response.Error.Message) + } + spanEnd(&err) + return response } // unsubscribe is the callback function for all *_unsubscribe calls. @@ -612,8 +660,11 @@ type limitedBuffer struct { } func (buf *limitedBuffer) Write(data []byte) (int, error) { - avail := max(buf.limit, len(buf.output)) - if len(data) < avail { + avail := buf.limit - len(buf.output) + if avail <= 0 { + return 0, errTruncatedOutput + } + if len(data) <= avail { buf.output = append(buf.output, data...) return len(data), nil } diff --git a/rpc/http.go b/rpc/http.go index f4b99429ef..55f0abfa72 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -30,6 +30,9 @@ import ( "strconv" "sync" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) const ( @@ -168,13 +171,21 @@ func newClientTransportHTTP(endpoint string, cfg *clientConfig) reconnectFunc { } } +// cleanlyCloseBody avoids sending unnecessary RST_STREAM and PING frames by +// ensuring the whole body is read before being closed. +// See https://blog.cloudflare.com/go-and-enhance-your-calm/#reading-bodies-in-go-can-be-unintuitive +func cleanlyCloseBody(body io.ReadCloser) error { + io.Copy(io.Discard, body) + return body.Close() +} + func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { hc := c.writeConn.(*httpConn) respBody, err := hc.doRequest(ctx, msg) if err != nil { return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var resp jsonrpcMessage batch := [1]*jsonrpcMessage{&resp} @@ -191,7 +202,7 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err != nil { return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var respmsgs []*jsonrpcMessage if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { @@ -236,7 +247,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - resp.Body.Close() + cleanlyCloseBody(resp.Body) return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, @@ -326,6 +337,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() ctx = context.WithValue(ctx, peerInfoContextKey{}, connInfo) + // Extract trace context from incoming headers. + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + // All checks passed, create a codec that reads directly from the request body // until EOF, writes the response to w, and orders the server to process a // single request. diff --git a/rpc/http_test.go b/rpc/http_test.go index 6c268b6292..15ddd59bd0 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -106,7 +106,7 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body if err != nil { t.Fatalf("request failed: %v", err) } - resp.Body.Close() + cleanlyCloseBody(resp.Body) confirmStatusCode(t, resp.StatusCode, expectedStatusCode) } diff --git a/rpc/server.go b/rpc/server.go index 5c9f54955f..a0a718df44 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/log" + "go.opentelemetry.io/otel/trace" ) const MetadataApi = "rpc" @@ -55,15 +56,17 @@ type Server struct { batchResponseLimit int httpBodyLimit int wsReadLimit int64 + tracerProvider trace.TracerProvider } // NewServer creates a new server instance with no registered handlers. func NewServer() *Server { server := &Server{ - idgen: randomIDGenerator(), - codecs: make(map[ServerCodec]struct{}), - httpBodyLimit: defaultBodyLimit, - wsReadLimit: wsDefaultReadLimit, + idgen: randomIDGenerator(), + codecs: make(map[ServerCodec]struct{}), + httpBodyLimit: defaultBodyLimit, + wsReadLimit: wsDefaultReadLimit, + tracerProvider: nil, } server.run.Store(true) // Register the default service providing meta information about the RPC service such @@ -133,6 +136,15 @@ func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) { c.Close() } +// setTracerProvider configures the OpenTelemetry TracerProvider for RPC call tracing. +// Note: This method (and the TracerProvider field in the Server/Handler struct) is +// primarily intended for testing. In particular, it allows tests to configure an +// isolated TracerProvider without changing the global provider, avoiding +// interference between tests running in parallel. +func (s *Server) setTracerProvider(tp trace.TracerProvider) { + s.tracerProvider = tp +} + func (s *Server) trackCodec(codec ServerCodec) bool { s.mutex.Lock() defer s.mutex.Unlock() @@ -160,7 +172,7 @@ func (s *Server) serveSingleRequest(ctx context.Context, codec ServerCodec) { return } - h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit) + h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit, s.tracerProvider) h.allowSubscribe = false defer h.close(io.EOF, nil) diff --git a/rpc/service.go b/rpc/service.go index 7cd7ca4297..c3e3bd1d05 100644 --- a/rpc/service.go +++ b/rpc/service.go @@ -100,14 +100,14 @@ func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { } // callback returns the callback corresponding to the given RPC method name. -func (r *serviceRegistry) callback(method string) *callback { +func (r *serviceRegistry) callback(method string) (cb *callback, service, methodName string) { before, after, found := strings.Cut(method, serviceMethodSeparator) if !found { - return nil + return nil, "", "" } r.mu.Lock() defer r.mu.Unlock() - return r.services[before].callbacks[after] + return r.services[before].callbacks[after], before, after } // subscription returns a subscription callback in the given service. diff --git a/rpc/tracing_test.go b/rpc/tracing_test.go new file mode 100644 index 0000000000..5a04c901fd --- /dev/null +++ b/rpc/tracing_test.go @@ -0,0 +1,268 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "context" + "net/http/httptest" + "testing" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +// attributeMap converts a slice of attributes to a map. +func attributeMap(attrs []attribute.KeyValue) map[string]string { + m := make(map[string]string) + for _, a := range attrs { + switch a.Value.Type() { + case attribute.STRING: + m[string(a.Key)] = a.Value.AsString() + case attribute.BOOL: + if a.Value.AsBool() { + m[string(a.Key)] = "true" + } else { + m[string(a.Key)] = "false" + } + default: + m[string(a.Key)] = a.Value.Emit() + } + } + return m +} + +// newTracingServer creates a new server with tracing enabled. +func newTracingServer(t *testing.T) (*Server, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + server := newTestServer() + server.setTracerProvider(tp) + t.Cleanup(server.Stop) + return server, tp, exporter +} + +// TestTracingHTTP verifies that RPC spans are emitted when processing HTTP requests. +func TestTracingHTTP(t *testing.T) { + // Not parallel: this test modifies the global otel TextMapPropagator. + + // Set up a propagator to extract W3C Trace Context headers. + originalPropagator := otel.GetTextMapPropagator() + otel.SetTextMapPropagator(propagation.TraceContext{}) + t.Cleanup(func() { otel.SetTextMapPropagator(originalPropagator) }) + + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + + // Define the expected trace and span IDs for context propagation. + const ( + traceID = "4bf92f3577b34da6a3ce929d0e0e4736" + parentSpanID = "00f067aa0ba902b7" + traceparent = "00-" + traceID + "-" + parentSpanID + "-01" + ) + + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Set trace context headers. + client.SetHeader("traceparent", traceparent) + + // Make a successful RPC call. + var result echoResult + if err := client.Call(&result, "test_echo", "hello", 42, &echoArgs{S: "world"}); err != nil { + t.Fatalf("RPC call failed: %v", err) + } + + // Flush and verify that we emitted the expected span. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + var rpcSpan *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "jsonrpc.test/echo" { + rpcSpan = &spans[i] + break + } + } + if rpcSpan == nil { + t.Fatalf("jsonrpc.test/echo span not found") + } + + // Verify span attributes. + attrs := attributeMap(rpcSpan.Attributes) + if attrs["rpc.system"] != "jsonrpc" { + t.Errorf("expected rpc.system=jsonrpc, got %v", attrs["rpc.system"]) + } + if attrs["rpc.service"] != "test" { + t.Errorf("expected rpc.service=test, got %v", attrs["rpc.service"]) + } + if attrs["rpc.method"] != "echo" { + t.Errorf("expected rpc.method=echo, got %v", attrs["rpc.method"]) + } + if _, ok := attrs["rpc.jsonrpc.request_id"]; !ok { + t.Errorf("expected rpc.jsonrpc.request_id attribute to be set") + } + + // Verify the span's parent matches the traceparent header values. + if got := rpcSpan.Parent.TraceID().String(); got != traceID { + t.Errorf("parent trace ID mismatch: got %s, want %s", got, traceID) + } + if got := rpcSpan.Parent.SpanID().String(); got != parentSpanID { + t.Errorf("parent span ID mismatch: got %s, want %s", got, parentSpanID) + } + if !rpcSpan.Parent.IsRemote() { + t.Error("expected parent span context to be marked as remote") + } +} + +// TestTracingErrorRecording verifies that errors are recorded on spans. +func TestTracingHTTPErrorRecording(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Call a method that returns an error. + var result any + err = client.Call(&result, "test_returnError") + if err == nil { + t.Fatal("expected error from test_returnError") + } + + // Flush and verify spans recorded the error. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + + // Only the runMethod span should have error status. + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + for _, span := range spans { + switch span.Name { + case "rpc.runMethod": + if span.Status.Code != codes.Error { + t.Errorf("expected %s span status Error, got %v", span.Name, span.Status.Code) + } + default: + if span.Status.Code == codes.Error { + t.Errorf("unexpected error status on span %s", span.Name) + } + } + } +} + +// TestTracingBatchHTTP verifies that RPC spans are emitted for batched JSON-RPC calls over HTTP. +func TestTracingBatchHTTP(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Make a successful batch RPC call. + batch := []BatchElem{ + { + Method: "test_echo", + Args: []any{"hello", 42, &echoArgs{S: "world"}}, + Result: new(echoResult), + }, + { + Method: "test_echo", + Args: []any{"your", 7, &echoArgs{S: "mom"}}, + Result: new(echoResult), + }, + } + if err := client.BatchCall(batch); err != nil { + t.Fatalf("batch RPC call failed: %v", err) + } + + // Flush and verify we emitted spans for each batch element. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + var found int + for i := range spans { + if spans[i].Name == "jsonrpc.test/echo" { + attrs := attributeMap(spans[i].Attributes) + if attrs["rpc.system"] == "jsonrpc" && + attrs["rpc.service"] == "test" && + attrs["rpc.method"] == "echo" && + attrs["rpc.batch"] == "true" { + found++ + } + } + } + if found != len(batch) { + t.Fatalf("expected %d matching batch spans, got %d", len(batch), found) + } +} + +// TestTracingSubscribeUnsubscribe verifies that subscribe and unsubscribe calls +// do not emit any spans. +// Note: This works because client.newClientConn() passes nil as the tracer provider. +func TestTracingSubscribeUnsubscribe(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + client := DialInProc(server) + t.Cleanup(client.Close) + + // Subscribe to notifications. + sub, err := client.Subscribe(context.Background(), "nftest", make(chan int), "someSubscription", 1, 1) + if err != nil { + t.Fatalf("subscribe failed: %v", err) + } + + // Unsubscribe. + sub.Unsubscribe() + + // Flush and check that no spans were emitted. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) != 0 { + t.Errorf("expected no spans for subscribe/unsubscribe, got %d", len(spans)) + } +} diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 19323246a7..401f4fba07 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -242,8 +242,8 @@ func (args *SendTxArgs) validateTxSidecar() error { if args.Proofs != nil { if len(args.Proofs) == n { // v1 transaction - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } @@ -259,25 +259,27 @@ func (args *SendTxArgs) validateTxSidecar() error { if args.Commitments == nil { // Generate commitment and proof. commitments := make([]kzg4844.Commitment, n) - proofs := make([]kzg4844.Proof, 0, n) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } commitments[i] = c } - if args.BlobVersion == 1 { - for i, b := range args.Blobs { - p, err := kzg4844.ComputeCellProofs(&b) + var proofs []kzg4844.Proof + if args.BlobVersion == types.BlobSidecarVersion1 { + proofs = make([]kzg4844.Proof, 0, n*kzg4844.CellProofsPerBlob) + for i := range args.Blobs { + p, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing cell proof: %v", i, err) } proofs = append(proofs, p...) } } else { - for i, b := range args.Blobs { - p, err := kzg4844.ComputeBlobProof(&b, commitments[i]) + proofs = make([]kzg4844.Proof, 0, n) + for i := range args.Blobs { + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], commitments[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } diff --git a/signer/core/apitypes/types_test.go b/signer/core/apitypes/types_test.go index 3e8f3bd43b..f010cef207 100644 --- a/signer/core/apitypes/types_test.go +++ b/signer/core/apitypes/types_test.go @@ -20,7 +20,6 @@ import ( "crypto/rand" "crypto/sha256" "encoding/json" - "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -308,7 +307,7 @@ func TestValidateTxSidecar(t *testing.T) { name: "blobs with v1 version flag - should generate cell proofs", args: SendTxArgs{ Blobs: []kzg4844.Blob{blob1}, - BlobVersion: 1, + BlobVersion: types.BlobSidecarVersion1, }, wantErr: false, }, @@ -366,6 +365,17 @@ func TestValidateTxSidecar(t *testing.T) { }, wantErr: true, }, + { + name: "invalid proof", + args: SendTxArgs{ + BlobVersion: types.BlobSidecarVersion1, + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + Proofs: []kzg4844.Proof{proof1, proof2}, // wrong proof + BlobHashes: []common.Hash{hash1}, + }, + wantErr: true, + }, } for _, tt := range tests { @@ -390,7 +400,6 @@ func TestValidateTxSidecar(t *testing.T) { t.Errorf("validateTxSidecar() should have generated commitments") } if args.Proofs == nil || (len(args.Proofs) != len(args.Blobs) && len(args.Proofs) != len(args.Blobs)*kzg4844.CellProofsPerBlob) { - fmt.Println("proofs", args.Proofs) t.Errorf("validateTxSidecar() should have generated proofs") } if args.BlobHashes == nil || len(args.BlobHashes) != len(args.Blobs) { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 1a70bcce46..2e0f422e56 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -116,27 +116,29 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t if !ok { return UnsupportedForkError{t.json.Network} } + // import pre accounts & construct test genesis block & state root + // Commit genesis state var ( + gspec = t.genesis(config) db = rawdb.NewMemoryDatabase() tconf = &triedb.Config{ Preimages: true, + IsVerkle: gspec.Config.VerkleTime != nil && *gspec.Config.VerkleTime <= gspec.Timestamp, } ) - if scheme == rawdb.PathScheme { + if scheme == rawdb.PathScheme || tconf.IsVerkle { tconf.PathDB = pathdb.Defaults } else { tconf.HashDB = hashdb.Defaults } - // Commit genesis state - gspec := t.genesis(config) // if ttd is not specified, set an arbitrary huge value if gspec.Config.TerminalTotalDifficulty == nil { gspec.Config.TerminalTotalDifficulty = big.NewInt(stdmath.MaxInt64) } triedb := triedb.NewDatabase(db, tconf) - gblock, err := gspec.Commit(db, triedb) + gblock, err := gspec.Commit(db, triedb, nil) if err != nil { return err } @@ -202,6 +204,11 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t return t.validateImportedHeaders(chain, validBlocks) } +// Network returns the network/fork name for this test. +func (t *BlockTest) Network() string { + return t.json.Network +} + func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { return &core.Genesis{ Config: config, @@ -259,7 +266,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) } if b.BlockHeader == nil { if data, err := json.MarshalIndent(cb.Header(), "", " "); err == nil { - fmt.Fprintf(os.Stderr, "block (index %d) insertion should have failed due to: %v:\n%v\n", + fmt.Fprintf(os.Stdout, "block (index %d) insertion should have failed due to: %v:\n%v\n", bi, b.ExpectException, string(data)) } return nil, fmt.Errorf("block (index %d) insertion should have failed due to: %v", diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 4d94d31c0c..c60c9cb6e6 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -32,7 +32,6 @@ import ( type kv struct { k, v []byte - t bool } type fuzzer struct { @@ -62,8 +61,8 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { size := f.readInt() // Fill it with some fluff for i := byte(0); i < byte(size); i++ { - value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} - value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false} + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}} + value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}} trie.MustUpdate(value.k, value.v) trie.MustUpdate(value2.k, value2.v) vals[string(value.k)] = value @@ -76,7 +75,7 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { for i := 0; i < n; i++ { k := f.randBytes(32) v := f.randBytes(20) - value := &kv{k, v, false} + value := &kv{k, v} trie.MustUpdate(k, v) vals[string(k)] = value if f.exhausted { diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index c136253a62..3baff33dcc 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -78,7 +78,7 @@ func fuzz(input []byte) int { rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!! f := fetcher.NewTxFetcherForTests( - func(common.Hash) bool { return false }, + func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { return make([]error, len(txs)) }, diff --git a/tests/init.go b/tests/init.go index 705e929ae9..d10b47986c 100644 --- a/tests/init.go +++ b/tests/init.go @@ -720,6 +720,25 @@ var Forks = map[string]*params.ChainConfig{ BPO4: params.DefaultBPO4BlobConfig, }, }, + "Verkle": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + VerkleTime: u64(0), + }, } var bpo1BlobConfig = ¶ms.BlobConfig{ diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 4f881fcd41..bec8ec71c6 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -239,6 +239,20 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if err != nil { // Here, an error exists but it was expected. // We do not check the post state or logs. + // However, if the test defines a post state root, we should check it. + // In case of an error, the state is reverted to the snapshot, so we need to + // recalculate the root. + post := t.json.Post[subtest.Fork][subtest.Index] + if post.Root != (common.UnprefixedHash{}) { + config, _, err := GetChainConfig(subtest.Fork) + if err != nil { + return fmt.Errorf("failed to get chain config: %w", err) + } + root = st.StateDB.IntermediateRoot(config.IsEIP158(new(big.Int).SetUint64(t.json.Env.Number))) + if root != common.Hash(post.Root) { + return fmt.Errorf("post-state root does not match the pre-state root, indicates an error in the test: got %x, want %x", root, post.Root) + } + } return nil } post := t.json.Post[subtest.Fork][subtest.Index] diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index 1c003a6c8f..690489b2aa 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -31,8 +31,11 @@ type ( var zero [32]byte const ( - NodeWidth = 256 // Number of child per leaf node - StemSize = 31 // Number of bytes to travel before reaching a group of leaves + StemNodeWidth = 256 // Number of child per leaf node + StemSize = 31 // Number of bytes to travel before reaching a group of leaves + NodeTypeBytes = 1 // Size of node type prefix in serialization + HashSize = 32 // Size of a hash in bytes + BitmapSize = 32 // Size of the bitmap in a stem node ) const ( @@ -58,25 +61,28 @@ type BinaryNode interface { func SerializeNode(node BinaryNode) []byte { switch n := (node).(type) { case *InternalNode: - var serialized [65]byte + // InternalNode: 1 byte type + 32 bytes left hash + 32 bytes right hash + var serialized [NodeTypeBytes + HashSize + HashSize]byte serialized[0] = nodeTypeInternal copy(serialized[1:33], n.left.Hash().Bytes()) copy(serialized[33:65], n.right.Hash().Bytes()) return serialized[:] case *StemNode: - var serialized [32 + 32 + 256*32]byte + // StemNode: 1 byte type + 31 bytes stem + 32 bytes bitmap + 256*32 bytes values + var serialized [NodeTypeBytes + StemSize + BitmapSize + StemNodeWidth*HashSize]byte serialized[0] = nodeTypeStem - copy(serialized[1:32], node.(*StemNode).Stem) - bitmap := serialized[32:64] - offset := 64 - for i, v := range node.(*StemNode).Values { + copy(serialized[NodeTypeBytes:NodeTypeBytes+StemSize], n.Stem) + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize + for i, v := range n.Values { if v != nil { bitmap[i/8] |= 1 << (7 - (i % 8)) - copy(serialized[offset:offset+32], v) - offset += 32 + copy(serialized[offset:offset+HashSize], v) + offset += HashSize } } - return serialized[:] + // Only return the actual data, not the entire array + return serialized[:offset] default: panic("invalid node type") } @@ -104,21 +110,21 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { if len(serialized) < 64 { return nil, invalidSerializedLength } - var values [256][]byte - bitmap := serialized[32:64] - offset := 64 + var values [StemNodeWidth][]byte + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize - for i := range 256 { + for i := range StemNodeWidth { if bitmap[i/8]>>(7-(i%8))&1 == 1 { - if len(serialized) < offset+32 { + if len(serialized) < offset+HashSize { return nil, invalidSerializedLength } - values[i] = serialized[offset : offset+32] - offset += 32 + values[i] = serialized[offset : offset+HashSize] + offset += HashSize } } return &StemNode{ - Stem: serialized[1:32], + Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], Values: values[:], depth: depth, }, nil diff --git a/trie/bintrie/binary_node_test.go b/trie/bintrie/binary_node_test.go index b21daaab69..242743ba53 100644 --- a/trie/bintrie/binary_node_test.go +++ b/trie/bintrie/binary_node_test.go @@ -77,12 +77,12 @@ func TestSerializeDeserializeInternalNode(t *testing.T) { // TestSerializeDeserializeStemNode tests serialization and deserialization of StemNode func TestSerializeDeserializeStemNode(t *testing.T) { // Create a stem node with some values - stem := make([]byte, 31) + stem := make([]byte, StemSize) for i := range stem { stem[i] = byte(i) } - var values [256][]byte + var values [StemNodeWidth][]byte // Add some values at different indices values[0] = common.HexToHash("0x0101010101010101010101010101010101010101010101010101010101010101").Bytes() values[10] = common.HexToHash("0x0202020202020202020202020202020202020202020202020202020202020202").Bytes() @@ -103,7 +103,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check the stem is correctly serialized - if !bytes.Equal(serialized[1:32], stem) { + if !bytes.Equal(serialized[1:1+StemSize], stem) { t.Errorf("Stem mismatch in serialized data") } @@ -136,7 +136,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check that other values are nil - for i := range NodeWidth { + for i := range StemNodeWidth { if i == 0 || i == 10 || i == 255 { continue } @@ -218,15 +218,15 @@ func TestKeyToPath(t *testing.T) { }, { name: "max valid depth", - depth: 31 * 8, - key: make([]byte, 32), - expected: make([]byte, 31*8+1), + depth: StemSize * 8, + key: make([]byte, HashSize), + expected: make([]byte, StemSize*8+1), wantErr: false, }, { name: "depth too large", - depth: 31*8 + 1, - key: make([]byte, 32), + depth: StemSize*8 + 1, + key: make([]byte, HashSize), wantErr: true, }, } diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index 8f9fd66a59..e4d8c2e7ac 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -46,8 +46,31 @@ func (h HashedNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error return nil, errors.New("attempted to get values from an unresolved node") } -func (h HashedNode) InsertValuesAtStem(key []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - return nil, errors.New("insertValuesAtStem not implemented for hashed node") +func (h HashedNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { + // Step 1: Generate the path for this node's position in the tree + path, err := keyToPath(depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem path generation error: %w", err) + } + + if resolver == nil { + return nil, errors.New("InsertValuesAtStem resolve error: resolver is nil") + } + + // Step 2: Resolve the hashed node to get the actual node data + data, err := resolver(path, common.Hash(h)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + + // Step 3: Deserialize the resolved data into a concrete node + node, err := DeserializeNode(data, depth) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + + // Step 4: Call InsertValuesAtStem on the resolved concrete node + return node.InsertValuesAtStem(stem, values, resolver, depth) } func (h HashedNode) toDot(parent string, path string) string { @@ -58,7 +81,8 @@ func (h HashedNode) toDot(parent string, path string) string { } func (h HashedNode) CollectNodes([]byte, NodeFlushFn) error { - return errors.New("collectNodes not implemented for hashed node") + // HashedNodes are already persisted in the database and don't need to be collected. + return nil } func (h HashedNode) GetHeight() int { diff --git a/trie/bintrie/hashed_node_test.go b/trie/bintrie/hashed_node_test.go index 0c19ae0c57..f9e6984888 100644 --- a/trie/bintrie/hashed_node_test.go +++ b/trie/bintrie/hashed_node_test.go @@ -17,6 +17,7 @@ package bintrie import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/common" @@ -59,8 +60,8 @@ func TestHashedNodeCopy(t *testing.T) { func TestHashedNodeInsert(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - key := make([]byte, 32) - value := make([]byte, 32) + key := make([]byte, HashSize) + value := make([]byte, HashSize) _, err := node.Insert(key, value, nil, 0) if err == nil { @@ -76,7 +77,7 @@ func TestHashedNodeInsert(t *testing.T) { func TestHashedNodeGetValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) + stem := make([]byte, StemSize) _, err := node.GetValuesAtStem(stem, nil) if err == nil { t.Fatal("Expected error for GetValuesAtStem on HashedNode") @@ -91,17 +92,85 @@ func TestHashedNodeGetValuesAtStem(t *testing.T) { func TestHashedNodeInsertValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) - values := make([][]byte, 256) + stem := make([]byte, StemSize) + values := make([][]byte, StemNodeWidth) + // Test 1: nil resolver should return an error _, err := node.InsertValuesAtStem(stem, values, nil, 0) if err == nil { - t.Fatal("Expected error for InsertValuesAtStem on HashedNode") + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with nil resolver") } - if err.Error() != "insertValuesAtStem not implemented for hashed node" { + if err.Error() != "InsertValuesAtStem resolve error: resolver is nil" { t.Errorf("Unexpected error message: %v", err) } + + // Test 2: mock resolver returning invalid data should return deserialization error + mockResolver := func(path []byte, hash common.Hash) ([]byte, error) { + // Return invalid/nonsense data that cannot be deserialized + return []byte{0xff, 0xff, 0xff}, nil + } + + _, err = node.InsertValuesAtStem(stem, values, mockResolver, 0) + if err == nil { + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with invalid resolver data") + } + + expectedPrefix := "InsertValuesAtStem node deserialization error:" + if len(err.Error()) < len(expectedPrefix) || err.Error()[:len(expectedPrefix)] != expectedPrefix { + t.Errorf("Expected deserialization error, got: %v", err) + } + + // Test 3: mock resolver returning valid serialized node should succeed + stem = make([]byte, StemSize) + stem[0] = 0xaa + var originalValues [StemNodeWidth][]byte + originalValues[0] = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111").Bytes() + originalValues[1] = common.HexToHash("0x2222222222222222222222222222222222222222222222222222222222222222").Bytes() + + originalNode := &StemNode{ + Stem: stem, + Values: originalValues[:], + depth: 0, + } + + // Serialize the node + serialized := SerializeNode(originalNode) + + // Create a mock resolver that returns the serialized node + validResolver := func(path []byte, hash common.Hash) ([]byte, error) { + return serialized, nil + } + + var newValues [StemNodeWidth][]byte + newValues[2] = common.HexToHash("0x3333333333333333333333333333333333333333333333333333333333333333").Bytes() + + resolvedNode, err := node.InsertValuesAtStem(stem, newValues[:], validResolver, 0) + if err != nil { + t.Fatalf("Expected successful resolution and insertion, got error: %v", err) + } + + resultStem, ok := resolvedNode.(*StemNode) + if !ok { + t.Fatalf("Expected resolved node to be *StemNode, got %T", resolvedNode) + } + + if !bytes.Equal(resultStem.Stem, stem) { + t.Errorf("Stem mismatch: expected %x, got %x", stem, resultStem.Stem) + } + + // Verify the original values are preserved + if !bytes.Equal(resultStem.Values[0], originalValues[0]) { + t.Errorf("Original value at index 0 not preserved: expected %x, got %x", originalValues[0], resultStem.Values[0]) + } + if !bytes.Equal(resultStem.Values[1], originalValues[1]) { + t.Errorf("Original value at index 1 not preserved: expected %x, got %x", originalValues[1], resultStem.Values[1]) + } + + // Verify the new value was inserted + if !bytes.Equal(resultStem.Values[2], newValues[2]) { + t.Errorf("New value at index 2 not inserted correctly: expected %x, got %x", newValues[2], resultStem.Values[2]) + } } // TestHashedNodeToDot tests the toDot method for visualization diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index f3ddd1aab0..0a7bece521 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -49,14 +49,26 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ } bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - var child *BinaryNode if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + return bt.left.GetValuesAtStem(stem, resolver) } - if hn, ok := (*child).(HashedNode); ok { + if hn, ok := bt.right.(HashedNode); ok { path, err := keyToPath(bt.depth, stem) if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) @@ -69,9 +81,9 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } - *child = node + bt.right = node } - return (*child).GetValuesAtStem(stem, resolver) + return bt.right.GetValuesAtStem(stem, resolver) } // Get retrieves the value for the given key. @@ -80,6 +92,9 @@ func (bt *InternalNode) Get(key []byte, resolver NodeResolverFn) ([]byte, error) if err != nil { return nil, fmt.Errorf("get error: %w", err) } + if values == nil { + return nil, nil + } return values[key[31]], nil } @@ -118,17 +133,54 @@ func (bt *InternalNode) Hash() common.Hash { // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - var ( - child *BinaryNode - err error - ) + var err error bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if bt.left == nil { + bt.left = Empty{} + } + + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + + bt.left, err = bt.left.InsertValuesAtStem(stem, values, resolver, depth+1) + return bt, err + } + + if bt.right == nil { + bt.right = Empty{} } - *child, err = (*child).InsertValuesAtStem(stem, values, resolver, depth+1) + + if hn, ok := bt.right.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.right = node + } + + bt.right, err = bt.right.InsertValuesAtStem(stem, values, resolver, depth+1) return bt, err } diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index a6bab2bcfa..9b863ed1e3 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -108,6 +108,11 @@ func (it *binaryNodeIterator) Next(descend bool) bool { } // go back to parent to get the next leaf + // Check if we're at the root before popping + if len(it.stack) == 1 { + it.lastErr = errIteratorEnd + return false + } it.stack = it.stack[:len(it.stack)-1] it.current = it.stack[len(it.stack)-1].Node it.stack[len(it.stack)-1].Index++ @@ -183,9 +188,31 @@ func (it *binaryNodeIterator) NodeBlob() []byte { } // Leaf returns true iff the current node is a leaf node. +// In a Binary Trie, a StemNode contains up to 256 leaf values. +// The iterator is only considered to be "at a leaf" when it's positioned +// at a specific non-nil value within the StemNode, not just at the StemNode itself. func (it *binaryNodeIterator) Leaf() bool { - _, ok := it.current.(*StemNode) - return ok + sn, ok := it.current.(*StemNode) + if !ok { + return false + } + + // Check if we have a valid stack position + if len(it.stack) == 0 { + return false + } + + // The Index in the stack state points to the NEXT position after the current value. + // So if Index is 0, we haven't started iterating through the values yet. + // If Index is 5, we're currently at value[4] (the 5th value, 0-indexed). + idx := it.stack[len(it.stack)-1].Index + if idx == 0 || idx > 256 { + return false + } + + // Check if there's actually a value at the current position + currentValueIndex := idx - 1 + return sn.Values[currentValueIndex] != nil } // LeafKey returns the key of the leaf. The method panics if the iterator is not @@ -219,7 +246,7 @@ func (it *binaryNodeIterator) LeafProof() [][]byte { panic("LeafProof() called on an binary node iterator not at a leaf location") } - proof := make([][]byte, 0, len(it.stack)+NodeWidth) + proof := make([][]byte, 0, len(it.stack)+StemNodeWidth) // Build proof by walking up the stack and collecting sibling hashes for i := range it.stack[:len(it.stack)-2] { diff --git a/trie/bintrie/iterator_test.go b/trie/bintrie/iterator_test.go deleted file mode 100644 index 8773e9e0c5..0000000000 --- a/trie/bintrie/iterator_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bintrie - -import ( - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-ethereum/triedb/hashdb" - "github.com/ethereum/go-ethereum/triedb/pathdb" - "github.com/holiman/uint256" -) - -func newTestDatabase(diskdb ethdb.Database, scheme string) *triedb.Database { - config := &triedb.Config{Preimages: true} - if scheme == rawdb.HashScheme { - config.HashDB = &hashdb.Config{CleanCacheSize: 0} - } else { - config.PathDB = &pathdb.Config{TrieCleanSize: 0, StateCleanSize: 0} - } - return triedb.NewDatabase(diskdb, config) -} - -func TestBinaryIterator(t *testing.T) { - trie, err := NewBinaryTrie(types.EmptyVerkleHash, newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)) - if err != nil { - t.Fatal(err) - } - account0 := &types.StateAccount{ - Nonce: 1, - Balance: uint256.NewInt(2), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // NOTE: the code size isn't written to the trie via TryUpdateAccount - // so it will be missing from the test nodes. - trie.UpdateAccount(common.Address{}, account0, 0) - account1 := &types.StateAccount{ - Nonce: 1337, - Balance: uint256.NewInt(2000), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // This address is meant to hash to a value that has the same first byte as 0xbf - var clash = common.HexToAddress("69fd8034cdb20934dedffa7dccb4fb3b8062a8be") - trie.UpdateAccount(clash, account1, 0) - - // Manually go over every node to check that we get all - // the correct nodes. - it, err := trie.NodeIterator(nil) - if err != nil { - t.Fatal(err) - } - var leafcount int - for it.Next(true) { - t.Logf("Node: %x", it.Path()) - if it.Leaf() { - leafcount++ - t.Logf("\tLeaf: %x", it.LeafKey()) - } - } - if leafcount != 2 { - t.Fatalf("invalid leaf count: %d != 6", leafcount) - } -} diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 13c2057371..94a22d52d0 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -33,8 +33,17 @@ const ( ) var ( - zeroHash = common.Hash{} - codeOffset = uint256.NewInt(128) + zeroInt = uint256.NewInt(0) + zeroHash = common.Hash{} + verkleNodeWidthLog2 = 8 + headerStorageOffset = uint256.NewInt(64) + codeOffset = uint256.NewInt(128) + codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) + mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) + CodeOffset = uint256.NewInt(128) + VerkleNodeWidth = uint256.NewInt(256) + HeaderStorageOffset = uint256.NewInt(64) + VerkleNodeWidthLog2 = 8 ) func GetBinaryTreeKey(addr common.Address, key []byte) []byte { @@ -42,11 +51,18 @@ func GetBinaryTreeKey(addr common.Address, key []byte) []byte { hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) hasher.Write(key[:31]) + hasher.Write([]byte{0}) k := hasher.Sum(nil) k[31] = key[31] return k } +func GetBinaryTreeKeyBasicData(addr common.Address) []byte { + var k [32]byte + k[31] = BasicDataLeafKey + return GetBinaryTreeKey(addr, k[:]) +} + func GetBinaryTreeKeyCodeHash(addr common.Address) []byte { var k [32]byte k[31] = CodeHashLeafKey @@ -77,3 +93,38 @@ func GetBinaryTreeKeyCodeChunk(address common.Address, chunknr *uint256.Int) []b chunkOffset := new(uint256.Int).Add(codeOffset, chunknr).Bytes() return GetBinaryTreeKey(address, chunkOffset) } + +func StorageIndex(storageKey []byte) (*uint256.Int, byte) { + // If the storage slot is in the header, we need to add the header offset. + var key uint256.Int + key.SetBytes(storageKey) + if key.Cmp(codeStorageDelta) < 0 { + // This addition is always safe; it can't ever overflow since pos> (7 - (bt.depth % 8)) & 1 n := &InternalNode{depth: bt.depth} @@ -65,26 +65,26 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int } *other = Empty{} } else { - var values [256][]byte - values[key[31]] = value + var values [StemNodeWidth][]byte + values[key[StemSize]] = value *other = &StemNode{ - Stem: slices.Clone(key[:31]), + Stem: slices.Clone(key[:StemSize]), Values: values[:], depth: depth + 1, } } return n, nil } - if len(value) != 32 { + if len(value) != HashSize { return bt, errors.New("invalid insertion: value length") } - bt.Values[key[31]] = value + bt.Values[key[StemSize]] = value return bt, nil } // Copy creates a deep copy of the node. func (bt *StemNode) Copy() BinaryNode { - var values [256][]byte + var values [StemNodeWidth][]byte for i, v := range bt.Values { values[i] = slices.Clone(v) } @@ -102,7 +102,7 @@ func (bt *StemNode) GetHeight() int { // Hash returns the hash of the node. func (bt *StemNode) Hash() common.Hash { - var data [NodeWidth]common.Hash + var data [StemNodeWidth]common.Hash for i, v := range bt.Values { if v != nil { h := sha256.Sum256(v) @@ -112,7 +112,7 @@ func (bt *StemNode) Hash() common.Hash { h := sha256.New() for level := 1; level <= 8; level++ { - for i := range NodeWidth / (1 << level) { + for i := range StemNodeWidth / (1 << level) { h.Reset() if data[i*2] == (common.Hash{}) && data[i*2+1] == (common.Hash{}) { @@ -141,14 +141,17 @@ func (bt *StemNode) CollectNodes(path []byte, flush NodeFlushFn) error { } // GetValuesAtStem retrieves the group of values located at the given stem key. -func (bt *StemNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { +func (bt *StemNode) GetValuesAtStem(stem []byte, _ NodeResolverFn) ([][]byte, error) { + if !bytes.Equal(bt.Stem, stem) { + return nil, nil + } return bt.Values[:], nil } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { - if !bytes.Equal(bt.Stem, key[:31]) { + if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 n := &InternalNode{depth: bt.depth} @@ -174,7 +177,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv *other = Empty{} } else { *other = &StemNode{ - Stem: slices.Clone(key[:31]), + Stem: slices.Clone(key[:StemSize]), Values: values, depth: n.depth + 1, } @@ -206,7 +209,7 @@ func (bt *StemNode) toDot(parent, path string) string { // Key returns the full key for the given index. func (bt *StemNode) Key(i int) []byte { - var ret [32]byte + var ret [HashSize]byte copy(ret[:], bt.Stem) ret[StemSize] = byte(i) return ret[:] diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index e0ffd5c3c8..d8d6844427 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -251,27 +251,23 @@ func TestStemNodeGetValuesAtStem(t *testing.T) { } // Check that all values match - for i := 0; i < 256; i++ { + for i := range 256 { if !bytes.Equal(retrievedValues[i], values[i]) { t.Errorf("Value mismatch at index %d", i) } } - // GetValuesAtStem with different stem also returns the same values - // (implementation ignores the stem parameter) + // GetValuesAtStem with different stem should return nil differentStem := make([]byte, 31) differentStem[0] = 0xFF - retrievedValues2, err := node.GetValuesAtStem(differentStem, nil) + shouldBeNil, err := node.GetValuesAtStem(differentStem, nil) if err != nil { t.Fatalf("Failed to get values with different stem: %v", err) } - // Should still return the same values (stem is ignored) - for i := 0; i < 256; i++ { - if !bytes.Equal(retrievedValues2[i], values[i]) { - t.Errorf("Value mismatch at index %d with different stem", i) - } + if shouldBeNil != nil { + t.Error("Expected nil for different stem, got non-nil") } } diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index 0a8bd325f5..ecdd002331 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -33,6 +33,81 @@ import ( var errInvalidRootType = errors.New("invalid root type") +// ChunkedCode represents a sequence of HashSize-byte chunks of code (StemSize bytes of which +// are actual code, and NodeTypeBytes byte is the pushdata offset). +type ChunkedCode []byte + +// Copy the values here so as to avoid an import cycle +const ( + PUSH1 = byte(0x60) + PUSH32 = byte(0x7f) +) + +// ChunkifyCode generates the chunked version of an array representing EVM bytecode +// according to EIP-7864 specification. +// +// The code is divided into HashSize-byte chunks, where each chunk contains: +// - Byte 0: Metadata byte indicating the number of leading bytes that are PUSHDATA (0-StemSize) +// - Bytes 1-StemSize: Actual code bytes +// +// This format enables stateless clients to validate jump destinations within a chunk +// without requiring additional context. When a PUSH instruction's data spans multiple +// chunks, the metadata byte tells us how many bytes at the start of the chunk are +// part of the previous chunk's PUSH instruction data. +// +// For example: +// - If a chunk starts with regular code: metadata byte = 0 +// - If a PUSH32 instruction starts at byte 30 of chunk N: +// - Chunk N: normal, contains PUSH32 opcode + 1 byte of data +// - Chunk N+1: metadata = StemSize (entire chunk is PUSH data) +// - Chunk N+2: metadata = 1 (first byte is PUSH data, then normal code resumes) +// +// This chunking approach ensures that jump destination validity can be determined +// by examining only the chunk containing the potential JUMPDEST, making it ideal +// for stateless execution and verkle/binary tries. +// +// Reference: https://eips.ethereum.org/EIPS/eip-7864 +func ChunkifyCode(code []byte) ChunkedCode { + var ( + chunkOffset = 0 // offset in the chunk + chunkCount = len(code) / StemSize + codeOffset = 0 // offset in the code + ) + if len(code)%StemSize != 0 { + chunkCount++ + } + chunks := make([]byte, chunkCount*HashSize) + for i := 0; i < chunkCount; i++ { + // number of bytes to copy, StemSize unless the end of the code has been reached. + end := min(len(code), StemSize*(i+1)) + copy(chunks[i*HashSize+1:], code[StemSize*i:end]) // copy the code itself + + // chunk offset = taken from the last chunk. + if chunkOffset > StemSize { + // skip offset calculation if push data covers the whole chunk + chunks[i*HashSize] = StemSize + chunkOffset = 1 + continue + } + chunks[HashSize*i] = byte(chunkOffset) + chunkOffset = 0 + + // Check each instruction and update the offset it should be 0 unless + // a PUSH-N overflows. + for ; codeOffset < end; codeOffset++ { + if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { + codeOffset += int(code[codeOffset] - PUSH1 + 1) + if codeOffset+1 >= StemSize*(i+1) { + codeOffset++ + chunkOffset = codeOffset - StemSize*(i+1) + break + } + } + } + } + return chunks +} + // NewBinaryNode creates a new empty binary trie func NewBinaryNode() BinaryNode { return Empty{} @@ -114,7 +189,7 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error ) switch r := t.root.(type) { case *InternalNode: - values, err = r.GetValuesAtStem(key[:31], t.nodeResolver) + values, err = r.GetValuesAtStem(key[:StemSize], t.nodeResolver) case *StemNode: values = r.Values case Empty: @@ -161,15 +236,15 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error // not be modified by the caller. If a node was not found in the database, a // trie.MissingNodeError is returned. func (t *BinaryTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - return t.root.Get(GetBinaryTreeKey(addr, key), t.nodeResolver) + return t.root.Get(GetBinaryTreeKeyStorageSlot(addr, key), t.nodeResolver) } // UpdateAccount updates the account information for the given address. func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { var ( err error - basicData [32]byte - values = make([][]byte, NodeWidth) + basicData [HashSize]byte + values = make([][]byte, StemNodeWidth) stem = GetBinaryTreeKey(addr, zero[:]) ) binary.BigEndian.PutUint32(basicData[BasicDataCodeSizeOffset-1:], uint32(codeLen)) @@ -177,14 +252,14 @@ func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, // Because the balance is a max of 16 bytes, truncate // the extra values. This happens in devmode, where - // 0xff**32 is allocated to the developer account. + // 0xff**HashSize is allocated to the developer account. balanceBytes := acc.Balance.Bytes() // TODO: reduce the size of the allocation in devmode, then panic instead // of truncating. if len(balanceBytes) > 16 { balanceBytes = balanceBytes[16:] } - copy(basicData[32-len(balanceBytes):], balanceBytes[:]) + copy(basicData[HashSize-len(balanceBytes):], balanceBytes[:]) values[BasicDataLeafKey] = basicData[:] values[CodeHashLeafKey] = acc.CodeHash[:] @@ -205,11 +280,11 @@ func (t *BinaryTrie) UpdateStem(key []byte, values [][]byte) error { // database, a trie.MissingNodeError is returned. func (t *BinaryTrie) UpdateStorage(address common.Address, key, value []byte) error { k := GetBinaryTreeKeyStorageSlot(address, key) - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) + var v [HashSize]byte + if len(value) >= HashSize { + copy(v[:], value[:HashSize]) } else { - copy(v[32-len(value):], value[:]) + copy(v[HashSize-len(value):], value[:]) } root, err := t.root.Insert(k, v[:], t.nodeResolver, 0) if err != nil { @@ -227,8 +302,8 @@ func (t *BinaryTrie) DeleteAccount(addr common.Address) error { // DeleteStorage removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. func (t *BinaryTrie) DeleteStorage(addr common.Address, key []byte) error { - k := GetBinaryTreeKey(addr, key) - var zero [32]byte + k := GetBinaryTreeKeyStorageSlot(addr, key) + var zero [HashSize]byte root, err := t.root.Insert(k, zero[:], t.nodeResolver, 0) if err != nil { return fmt.Errorf("DeleteStorage (%x) error: %v", addr, err) @@ -246,12 +321,12 @@ func (t *BinaryTrie) Hash() common.Hash { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. func (t *BinaryTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*InternalNode) nodeset := trienode.NewNodeSet(common.Hash{}) - err := root.CollectNodes(nil, func(path []byte, node BinaryNode) { + // The root can be any type of BinaryNode (InternalNode, StemNode, etc.) + err := t.root.CollectNodes(nil, func(path []byte, node BinaryNode) { serialized := SerializeNode(node) - nodeset.AddNode(path, trienode.NewNodeWithPrev(common.Hash{}, serialized, t.tracer.Get(path))) + nodeset.AddNode(path, trienode.NewNodeWithPrev(node.Hash(), serialized, t.tracer.Get(path))) }) if err != nil { panic(fmt.Errorf("CollectNodes failed: %v", err)) @@ -299,23 +374,23 @@ func (t *BinaryTrie) IsVerkle() bool { // Note: the basic data leaf needs to have been previously created for this to work func (t *BinaryTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { var ( - chunks = trie.ChunkifyCode(code) + chunks = ChunkifyCode(code) values [][]byte key []byte err error ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 + for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+HashSize, chunknr+1 { + groupOffset := (chunknr + 128) % StemNodeWidth if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, NodeWidth) - var offset [32]byte + values = make([][]byte, StemNodeWidth) + var offset [HashSize]byte binary.LittleEndian.PutUint64(offset[24:], chunknr+128) key = GetBinaryTreeKey(addr, offset[:]) } - values[groupOffset] = chunks[i : i+32] + values[groupOffset] = chunks[i : i+HashSize] - if groupOffset == 255 || len(chunks)-i <= 32 { - err = t.UpdateStem(key[:31], values) + if groupOffset == StemNodeWidth-1 || len(chunks)-i <= HashSize { + err = t.UpdateStem(key[:StemSize], values) if err != nil { return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) @@ -349,5 +424,5 @@ func (t *BinaryTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { // Witness returns a set containing all trie nodes that have been accessed. func (t *BinaryTrie) Witness() map[string][]byte { - panic("not implemented") + return t.tracer.Values() } diff --git a/trie/bintrie/trie_test.go b/trie/bintrie/trie_test.go index 84f7689549..256fd218e2 100644 --- a/trie/bintrie/trie_test.go +++ b/trie/bintrie/trie_test.go @@ -22,10 +22,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( - zeroKey = [32]byte{} + zeroKey = [HashSize]byte{} oneKey = common.HexToHash("0101010101010101010101010101010101010101010101010101010101010101") twoKey = common.HexToHash("0202020202020202020202020202020202020202020202020202020202020202") threeKey = common.HexToHash("0303030303030303030303030303030303030303030303030303030303030303") @@ -158,8 +161,8 @@ func TestInsertDuplicateKey(t *testing.T) { func TestLargeNumberOfEntries(t *testing.T) { var err error tree := NewBinaryNode() - for i := range 256 { - var key [32]byte + for i := range StemNodeWidth { + var key [HashSize]byte key[0] = byte(i) tree, err = tree.Insert(key[:], ffKey[:], nil, 0) if err != nil { @@ -182,7 +185,7 @@ func TestMerkleizeMultipleEntries(t *testing.T) { common.HexToHash("8100000000000000000000000000000000000000000000000000000000000000").Bytes(), } for i, key := range keys { - var v [32]byte + var v [HashSize]byte binary.LittleEndian.PutUint64(v[:8], uint64(i)) tree, err = tree.Insert(key, v[:], nil, 0) if err != nil { @@ -195,3 +198,97 @@ func TestMerkleizeMultipleEntries(t *testing.T) { t.Fatalf("invalid root, expected=%x, got = %x", expected, got) } } + +// TestStorageRoundTrip verifies that GetStorage and DeleteStorage use the same +// key mapping as UpdateStorage (GetBinaryTreeKeyStorageSlot). This is a regression +// test: previously GetStorage and DeleteStorage used GetBinaryTreeKey directly, +// which produced different tree keys and broke the read/delete path. +func TestStorageRoundTrip(t *testing.T) { + tracer := trie.NewPrevalueTracer() + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + addr := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + + // Create an account first so the root becomes an InternalNode, + // which is the realistic state when storage operations happen. + acc := &types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + CodeHash: common.HexToHash("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + } + if err := tr.UpdateAccount(addr, acc, 0); err != nil { + t.Fatalf("UpdateAccount error: %v", err) + } + + // Test main storage slots (key[31] >= 64 or key[:31] != 0). + // These produce a different stem than the account data, so after + // UpdateAccount + UpdateStorage the root is an InternalNode. + // Note: header slots (key[31] < 64, key[:31] == 0) share the same + // stem as account data and are covered by GetAccount/UpdateAccount path. + slots := []common.Hash{ + common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), // main storage (slot 255) + common.HexToHash("0100000000000000000000000000000000000000000000000000000000000001"), // main storage (non-zero prefix) + } + val := common.TrimLeftZeroes(common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef").Bytes()) + + for _, slot := range slots { + // Write + if err := tr.UpdateStorage(addr, slot[:], val); err != nil { + t.Fatalf("UpdateStorage(%x) error: %v", slot, err) + } + // Read back + got, err := tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) error: %v", slot, err) + } + if len(got) == 0 { + t.Fatalf("GetStorage(%x) returned empty, expected value", slot) + } + // Verify value (right-justified in 32 bytes) + var expected [HashSize]byte + copy(expected[HashSize-len(val):], val) + if !bytes.Equal(got, expected[:]) { + t.Fatalf("GetStorage(%x) = %x, want %x", slot, got, expected) + } + // Delete + if err := tr.DeleteStorage(addr, slot[:]); err != nil { + t.Fatalf("DeleteStorage(%x) error: %v", slot, err) + } + // Verify deleted (should read as zero, not the old value) + got, err = tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) after delete error: %v", slot, err) + } + if len(got) > 0 && !bytes.Equal(got, zero[:]) { + t.Fatalf("GetStorage(%x) after delete = %x, expected zero", slot, got) + } + } +} + +func TestBinaryTrieWitness(t *testing.T) { + tracer := trie.NewPrevalueTracer() + + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + if w := tr.Witness(); len(w) != 0 { + t.Fatal("expected empty witness for fresh trie") + } + + tracer.Put([]byte("path1"), []byte("blob1")) + tracer.Put([]byte("path2"), []byte("blob2")) + + witness := tr.Witness() + if len(witness) != 2 { + t.Fatalf("expected 2 witness entries, got %d", len(witness)) + } + if !bytes.Equal(witness[string([]byte("path1"))], []byte("blob1")) { + t.Fatal("unexpected witness value for path1") + } + if !bytes.Equal(witness[string([]byte("path2"))], []byte("blob2")) { + t.Fatal("unexpected witness value for path2") + } +} diff --git a/trie/node.go b/trie/node.go index 74fac4fd4e..7022116048 100644 --- a/trie/node.go +++ b/trie/node.go @@ -17,6 +17,7 @@ package trie import ( + "bytes" "fmt" "io" "strings" @@ -160,11 +161,14 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("decode error: %v", err) } - switch c, _ := rlp.CountValues(elems); c { - case 2: + c, err := rlp.CountValues(elems) + switch { + case err != nil: + return nil, fmt.Errorf("invalid node list: %v", err) + case c == 2: n, err := decodeShort(hash, elems) return n, wrapError(err, "short") - case 17: + case c == 17: n, err := decodeFull(hash, elems) return n, wrapError(err, "full") default: @@ -224,7 +228,7 @@ func decodeRef(buf []byte) (node, []byte, error) { case kind == rlp.List: // 'embedded' node reference. The encoding must be smaller // than a hash in order to be valid. - if size := len(buf) - len(rest); size > hashLen { + if size := len(buf) - len(rest); size >= hashLen { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } @@ -242,6 +246,74 @@ func decodeRef(buf []byte) (node, []byte, error) { } } +// decodeNodeElements parses the RLP encoding of a trie node and returns all the +// elements in raw byte format. +// +// For full node, it returns a slice of 17 elements; +// For short node, it returns a slice of 2 elements; +func decodeNodeElements(buf []byte) ([][]byte, error) { + if len(buf) == 0 { + return nil, io.ErrUnexpectedEOF + } + return rlp.SplitListValues(buf) +} + +// encodeNodeElements encodes the provided node elements into a rlp list. +func encodeNodeElements(elements [][]byte) ([]byte, error) { + if len(elements) != 2 && len(elements) != 17 { + return nil, fmt.Errorf("invalid number of elements: %d", len(elements)) + } + return rlp.MergeListValues(elements) +} + +// NodeDifference accepts two RLP-encoding nodes and figures out the difference +// between them. +// +// An error is returned if any of the provided blob is nil, or the type of nodes +// are different. +func NodeDifference(oldvalue []byte, newvalue []byte) (int, []int, [][]byte, error) { + oldElems, err := decodeNodeElements(oldvalue) + if err != nil { + return 0, nil, nil, err + } + newElems, err := decodeNodeElements(newvalue) + if err != nil { + return 0, nil, nil, err + } + if len(oldElems) != len(newElems) { + return 0, nil, nil, fmt.Errorf("different node type, old elements: %d, new elements: %d", len(oldElems), len(newElems)) + } + var ( + indices = make([]int, 0, len(oldElems)) + diff = make([][]byte, 0, len(oldElems)) + ) + for i := 0; i < len(oldElems); i++ { + if !bytes.Equal(oldElems[i], newElems[i]) { + indices = append(indices, i) + diff = append(diff, oldElems[i]) + } + } + return len(oldElems), indices, diff, nil +} + +// ReassembleNode accepts a RLP-encoding node along with a set of mutations, +// applying the modification diffs according to the indices and re-assemble. +func ReassembleNode(blob []byte, mutations [][][]byte, indices [][]int) ([]byte, error) { + if len(mutations) == 0 && len(indices) == 0 { + return blob, nil + } + elements, err := decodeNodeElements(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(mutations); i++ { + for j, pos := range indices[i] { + elements[pos] = mutations[i][j] + } + } + return encodeNodeElements(elements) +} + // wraps a decoding error with information about the path to the // invalid child node (for debugging encoding issues). type decodeError struct { diff --git a/trie/node_test.go b/trie/node_test.go index 9b8b33748f..b4f427ca77 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -18,9 +18,13 @@ package trie import ( "bytes" + "math/rand" + "reflect" + "slices" "testing" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" ) @@ -94,6 +98,286 @@ func TestDecodeFullNode(t *testing.T) { } } +func makeTestLeafNode(small bool) []byte { + l := leafNodeEncoder{} + l.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + if small { + l.Val = testrand.Bytes(10) + } else { + l.Val = testrand.Bytes(32) + } + buf := rlp.NewEncoderBuffer(nil) + l.encode(buf) + return buf.ToBytes() +} + +func makeTestFullNode(small bool) []byte { + n := fullnodeEncoder{} + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write hash + n.Children[i] = testrand.Bytes(32) + case 2: + // write embedded node + n.Children[i] = makeTestLeafNode(small) + } + } + n.Children[16] = testrand.Bytes(32) // value + buf := rlp.NewEncoderBuffer(nil) + n.encode(buf) + return buf.ToBytes() +} + +func TestEncodeDecodeNodeElements(t *testing.T) { + var nodes [][]byte + nodes = append(nodes, makeTestFullNode(true)) + nodes = append(nodes, makeTestFullNode(false)) + nodes = append(nodes, makeTestLeafNode(true)) + nodes = append(nodes, makeTestLeafNode(false)) + + for _, blob := range nodes { + elements, err := decodeNodeElements(blob) + if err != nil { + t.Fatalf("Failed to decode node elements: %v", err) + } + enc, err := encodeNodeElements(elements) + if err != nil { + t.Fatalf("Failed to encode node elements: %v", err) + } + if !bytes.Equal(enc, blob) { + t.Fatalf("Unexpected encoded node element, want: %v, got: %v", blob, enc) + } + } +} + +func makeTestLeafNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = leafNodeEncoder{} + nb = leafNodeEncoder{} + ) + key := keybytesToHex(testrand.Bytes(10)) + na.Key = hexToCompact(key) + nb.Key = hexToCompact(key) + + valA := testrand.Bytes(32) + valB := testrand.Bytes(32) + na.Val = valA + nb.Val = valB + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + diff, _ := rlp.EncodeToBytes(valA) + return bufa.ToBytes(), bufb.ToBytes(), [][]byte{diff}, []int{1} +} + +func makeTestFullNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = fullnodeEncoder{} + nb = fullnodeEncoder{} + indices []int + values [][]byte + ) + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write same + var child []byte + if rand.Intn(2) == 0 { + child = testrand.Bytes(32) // hashnode + } else { + child = makeTestLeafNode(true) // embedded node + } + na.Children[i] = child + nb.Children[i] = child + case 2: + // write different + var ( + va []byte + diff []byte + ) + rnd := rand.Intn(3) + if rnd == 0 { + va = testrand.Bytes(32) // hashnode + diff, _ = rlp.EncodeToBytes(va) + } else if rnd == 1 { + va = makeTestLeafNode(true) // embedded node + diff = va + } else { + va = nil + diff = rlp.EmptyString + } + vb := testrand.Bytes(32) // hashnode + na.Children[i] = va + nb.Children[i] = vb + + indices = append(indices, i) + values = append(values, diff) + } + } + na.Children[16] = nil + nb.Children[16] = nil + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + return bufa.ToBytes(), bufb.ToBytes(), values, indices +} + +func TestNodeDifference(t *testing.T) { + type testsuite struct { + old []byte + new []byte + expErr bool + expIndices []int + expValues [][]byte + } + var tests = []testsuite{ + // Invalid node data + { + old: nil, new: nil, expErr: true, + }, + { + old: testrand.Bytes(32), new: nil, expErr: true, + }, + { + old: nil, new: testrand.Bytes(32), expErr: true, + }, + { + old: bytes.Repeat([]byte{0x1}, 32), new: bytes.Repeat([]byte{0x2}, 32), expErr: true, + }, + // Different node type + { + old: makeTestLeafNode(true), new: makeTestFullNode(true), expErr: true, + }, + } + for range 10 { + va, vb, elements, indices := makeTestLeafNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + for range 10 { + va, vb, elements, indices := makeTestFullNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + + for i, test := range tests { + _, indices, values, err := NodeDifference(test.old, test.new) + if test.expErr && err == nil { + t.Fatalf("Expect error, got nil %d", i) + } + if !test.expErr && err != nil { + t.Fatalf("Unexpect error, %v", err) + } + if err == nil { + if !slices.Equal(indices, test.expIndices) { + t.Fatalf("Unexpected indices, want: %v, got: %v", test.expIndices, indices) + } + if !slices.EqualFunc(values, test.expValues, bytes.Equal) { + t.Fatalf("Unexpected values, want: %v, got: %v", test.expValues, values) + } + } + } +} + +func TestReassembleFullNode(t *testing.T) { + var fn fullnodeEncoder + for i := 0; i < 16; i++ { + if rand.Intn(2) == 0 { + fn.Children[i] = testrand.Bytes(32) + } + } + buf := rlp.NewEncoderBuffer(nil) + fn.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + var ( + pos = make(map[int]struct{}) + poslist []int + valuelist [][]byte + ) + for j := 0; j < 3; j++ { + p := rand.Intn(16) + if _, ok := pos[p]; ok { + continue + } + pos[p] = struct{}{} + + nh := testrand.Bytes(32) + diff, _ := rlp.EncodeToBytes(nh) + poslist = append(poslist, p) + valuelist = append(valuelist, diff) + fn.Children[p] = nh + } + values = append(values, valuelist) + indices = append(indices, poslist) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + fn.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + +func TestReassembleShortNode(t *testing.T) { + var ln leafNodeEncoder + ln.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + ln.Val = testrand.Bytes(10) + buf := rlp.NewEncoderBuffer(nil) + ln.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + val := testrand.Bytes(10) + ln.Val = val + diff, _ := rlp.EncodeToBytes(val) + values = append(values, [][]byte{diff}) + indices = append(indices, []int{1}) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + ln.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + // goos: darwin // goarch: arm64 // pkg: github.com/ethereum/go-ethereum/trie diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 7c7bd184bf..1f150ede8c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -134,9 +134,9 @@ func (t *StateTrie) GetAccountByHash(addrHash common.Hash) (*types.StateAccount, // PrefetchAccount attempts to resolve specific accounts from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchAccount(addresses []common.Address) error { - var keys [][]byte - for _, addr := range addresses { - keys = append(keys, crypto.Keccak256(addr.Bytes())) + keys := make([][]byte, len(addresses)) + for i, addr := range addresses { + keys[i] = crypto.Keccak256(addr.Bytes()) } return t.trie.Prefetch(keys) } @@ -157,9 +157,9 @@ func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { // PrefetchStorage attempts to resolve specific storage slots from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchStorage(_ common.Address, keys [][]byte) error { - var keylist [][]byte - for _, key := range keys { - keylist = append(keylist, crypto.Keccak256(key)) + keylist := make([][]byte, len(keys)) + for i, key := range keys { + keylist[i] = crypto.Keccak256(key) } return t.trie.Prefetch(keylist) } diff --git a/trie/transition.go b/trie/transitiontrie/transition.go similarity index 87% rename from trie/transition.go rename to trie/transitiontrie/transition.go index c6eecd3937..4c73022082 100644 --- a/trie/transition.go +++ b/trie/transitiontrie/transition.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package transitiontrie import ( "fmt" @@ -22,8 +22,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-verkle" ) // TransitionTrie is a trie that implements a façade design pattern, presenting @@ -31,13 +32,16 @@ import ( // first from the overlay trie, and falls back to the base trie if the key isn't // found. All writes go to the overlay trie. type TransitionTrie struct { - overlay *VerkleTrie - base *SecureTrie + overlay *bintrie.BinaryTrie + base *trie.SecureTrie storage bool } // NewTransitionTrie creates a new TransitionTrie. -func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie { +// Note: base can be nil when using TransitionTrie as a wrapper for BinaryTrie +// to work around import cycles. This is a temporary hack that should be +// refactored in future PRs (see core/state/reader.go for details). +func NewTransitionTrie(base *trie.SecureTrie, overlay *bintrie.BinaryTrie, st bool) *TransitionTrie { return &TransitionTrie{ overlay: overlay, base: base, @@ -46,12 +50,12 @@ func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *Transiti } // Base returns the base trie. -func (t *TransitionTrie) Base() *SecureTrie { +func (t *TransitionTrie) Base() *trie.SecureTrie { return t.base } // Overlay returns the overlay trie. -func (t *TransitionTrie) Overlay() *VerkleTrie { +func (t *TransitionTrie) Overlay() *bintrie.BinaryTrie { return t.overlay } @@ -61,7 +65,10 @@ func (t *TransitionTrie) GetKey(key []byte) []byte { if key := t.overlay.GetKey(key); key != nil { return key } - return t.base.GetKey(key) + if t.base != nil { + return t.base.GetKey(key) + } + return nil } // GetStorage returns the value for key stored in the trie. The value bytes must @@ -74,8 +81,11 @@ func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, er if len(val) != 0 { return val, nil } - // TODO also insert value into overlay - return t.base.GetStorage(addr, key) + if t.base != nil { + // TODO also insert value into overlay + return t.base.GetStorage(addr, key) + } + return nil, nil } // PrefetchStorage attempts to resolve specific storage slots from the database @@ -102,7 +112,10 @@ func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount if data != nil { return data, nil } - return t.base.GetAccount(address) + if t.base != nil { + return t.base.GetAccount(address) + } + return nil, nil } // PrefetchAccount attempts to resolve specific accounts from the database @@ -174,7 +187,7 @@ func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSe // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. -func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) { +func (t *TransitionTrie) NodeIterator(startKey []byte) (trie.NodeIterator, error) { panic("not implemented") // TODO: Implement } @@ -197,14 +210,10 @@ func (t *TransitionTrie) IsVerkle() bool { // UpdateStem updates a group of values, given the stem they are using. If // a value already exists, it is overwritten. +// TODO: This is Verkle-specific and requires access to private fields. +// Not currently used in the codebase. func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error { - trie := t.overlay - switch root := trie.root.(type) { - case *verkle.InternalNode: - return root.InsertValuesAtStem(key, values, t.overlay.nodeResolver) - default: - panic("invalid root type") - } + panic("UpdateStem is not implemented for TransitionTrie") } // Copy creates a deep copy of the transition trie. diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go deleted file mode 100644 index dea210c046..0000000000 --- a/trie/utils/verkle.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package utils - -import ( - "encoding/binary" - "sync" - - "github.com/crate-crypto/go-ipa/bandersnatch/fr" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -const ( - BasicDataLeafKey = 0 - CodeHashLeafKey = 1 - - BasicDataVersionOffset = 0 - BasicDataCodeSizeOffset = 5 - BasicDataNonceOffset = 8 - BasicDataBalanceOffset = 16 -) - -var ( - zero = uint256.NewInt(0) - verkleNodeWidthLog2 = 8 - headerStorageOffset = uint256.NewInt(64) - codeOffset = uint256.NewInt(128) - verkleNodeWidth = uint256.NewInt(256) - codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) - mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) - - index0Point *verkle.Point // pre-computed commitment of polynomial [2+256*64] - - // cacheHitGauge is the metric to track how many cache hit occurred. - cacheHitGauge = metrics.NewRegisteredGauge("trie/verkle/cache/hit", nil) - - // cacheMissGauge is the metric to track how many cache miss occurred. - cacheMissGauge = metrics.NewRegisteredGauge("trie/verkle/cache/miss", nil) -) - -func init() { - // The byte array is the Marshalled output of the point computed as such: - // - // var ( - // config = verkle.GetConfig() - // fr verkle.Fr - // ) - // verkle.FromLEBytes(&fr, []byte{2, 64}) - // point := config.CommitToPoly([]verkle.Fr{fr}, 1) - index0Point = new(verkle.Point) - err := index0Point.SetBytes([]byte{34, 25, 109, 242, 193, 5, 144, 224, 76, 52, 189, 92, 197, 126, 9, 145, 27, 152, 199, 130, 165, 3, 210, 27, 193, 131, 142, 28, 110, 26, 16, 191}) - if err != nil { - panic(err) - } -} - -// PointCache is the LRU cache for storing evaluated address commitment. -type PointCache struct { - lru lru.BasicLRU[string, *verkle.Point] - lock sync.RWMutex -} - -// NewPointCache returns the cache with specified size. -func NewPointCache(maxItems int) *PointCache { - return &PointCache{ - lru: lru.NewBasicLRU[string, *verkle.Point](maxItems), - } -} - -// Get returns the cached commitment for the specified address, or computing -// it on the flight. -func (c *PointCache) Get(addr []byte) *verkle.Point { - c.lock.Lock() - defer c.lock.Unlock() - - p, ok := c.lru.Get(string(addr)) - if ok { - cacheHitGauge.Inc(1) - return p - } - cacheMissGauge.Inc(1) - p = evaluateAddressPoint(addr) - c.lru.Add(string(addr), p) - return p -} - -// GetStem returns the first 31 bytes of the tree key as the tree stem. It only -// works for the account metadata whose treeIndex is 0. -func (c *PointCache) GetStem(addr []byte) []byte { - p := c.Get(addr) - return pointToHash(p, 0)[:31] -} - -// GetTreeKey performs both the work of the spec's get_tree_key function, and that -// of pedersen_hash: it builds the polynomial in pedersen_hash without having to -// create a mostly zero-filled buffer and "type cast" it to a 128-long 16-byte -// array. Since at most the first 5 coefficients of the polynomial will be non-zero, -// these 5 coefficients are created directly. -func GetTreeKey(address []byte, treeIndex *uint256.Int, subIndex byte) []byte { - if len(address) < 32 { - var aligned [32]byte - address = append(aligned[:32-len(address)], address...) - } - // poly = [2+256*64, address_le_low, address_le_high, tree_index_le_low, tree_index_le_high] - var poly [5]fr.Element - - // 32-byte address, interpreted as two little endian - // 16-byte numbers. - verkle.FromLEBytes(&poly[1], address[:16]) - verkle.FromLEBytes(&poly[2], address[16:]) - - // treeIndex must be interpreted as a 32-byte aligned little-endian integer. - // e.g: if treeIndex is 0xAABBCC, we need the byte representation to be 0xCCBBAA00...00. - // poly[3] = LE({CC,BB,AA,00...0}) (16 bytes), poly[4]=LE({00,00,...}) (16 bytes). - // - // To avoid unnecessary endianness conversions for go-ipa, we do some trick: - // - poly[3]'s byte representation is the same as the *top* 16 bytes (trieIndexBytes[16:]) of - // 32-byte aligned big-endian representation (BE({00,...,AA,BB,CC})). - // - poly[4]'s byte representation is the same as the *low* 16 bytes (trieIndexBytes[:16]) of - // the 32-byte aligned big-endian representation (BE({00,00,...}). - trieIndexBytes := treeIndex.Bytes32() - verkle.FromBytes(&poly[3], trieIndexBytes[16:]) - verkle.FromBytes(&poly[4], trieIndexBytes[:16]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add a constant point corresponding to poly[0]=[2+256*64]. - ret.Add(ret, index0Point) - - return pointToHash(ret, subIndex) -} - -// GetTreeKeyWithEvaluatedAddress is basically identical to GetTreeKey, the only -// difference is a part of polynomial is already evaluated. -// -// Specifically, poly = [2+256*64, address_le_low, address_le_high] is already -// evaluated. -func GetTreeKeyWithEvaluatedAddress(evaluated *verkle.Point, treeIndex *uint256.Int, subIndex byte) []byte { - var poly [5]fr.Element - - // little-endian, 32-byte aligned treeIndex - var index [32]byte - for i := 0; i < len(treeIndex); i++ { - binary.LittleEndian.PutUint64(index[i*8:(i+1)*8], treeIndex[i]) - } - verkle.FromLEBytes(&poly[3], index[:16]) - verkle.FromLEBytes(&poly[4], index[16:]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add the pre-evaluated address - ret.Add(ret, evaluated) - - return pointToHash(ret, subIndex) -} - -// BasicDataKey returns the verkle tree key of the basic data field for -// the specified account. -func BasicDataKey(address []byte) []byte { - return GetTreeKey(address, zero, BasicDataLeafKey) -} - -// CodeHashKey returns the verkle tree key of the code hash field for -// the specified account. -func CodeHashKey(address []byte) []byte { - return GetTreeKey(address, zero, CodeHashLeafKey) -} - -func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) { - var ( - chunkOffset = new(uint256.Int).Add(codeOffset, chunk) - treeIndex, subIndexMod = new(uint256.Int).DivMod(chunkOffset, verkleNodeWidth, new(uint256.Int)) - ) - return treeIndex, byte(subIndexMod.Uint64()) -} - -// CodeChunkKey returns the verkle tree key of the code chunk for the -// specified account. -func CodeChunkKey(address []byte, chunk *uint256.Int) []byte { - treeIndex, subIndex := codeChunkIndex(chunk) - return GetTreeKey(address, treeIndex, subIndex) -} - -func StorageIndex(storageKey []byte) (*uint256.Int, byte) { - // If the storage slot is in the header, we need to add the header offset. - var key uint256.Int - key.SetBytes(storageKey) - if key.Cmp(codeStorageDelta) < 0 { - // This addition is always safe; it can't ever overflow since pos. - -package utils - -import ( - "bytes" - "testing" - - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -func TestTreeKey(t *testing.T) { - var ( - address = []byte{0x01} - addressEval = evaluateAddressPoint(address) - smallIndex = uint256.NewInt(1) - largeIndex = uint256.NewInt(10000) - smallStorage = []byte{0x1} - largeStorage = bytes.Repeat([]byte{0xff}, 16) - ) - if !bytes.Equal(BasicDataKey(address), BasicDataKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched basic data key") - } - if !bytes.Equal(CodeHashKey(address), CodeHashKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched code hash key") - } - if !bytes.Equal(CodeChunkKey(address, smallIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, smallIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(CodeChunkKey(address, largeIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, largeIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(StorageSlotKey(address, smallStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, smallStorage)) { - t.Fatal("Unmatched storage slot key") - } - if !bytes.Equal(StorageSlotKey(address, largeStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, largeStorage)) { - t.Fatal("Unmatched storage slot key") - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKey -// BenchmarkTreeKey-8 398731 2961 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - BasicDataKey([]byte{0x01}) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKeyWithEvaluation -// BenchmarkTreeKeyWithEvaluation-8 513855 2324 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - BasicDataKeyWithEvaluatedAddress(eval) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKey -// BenchmarkStorageKey-8 230516 4584 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - StorageSlotKey([]byte{0x01}, bytes.Repeat([]byte{0xff}, 32)) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKeyWithEvaluation -// BenchmarkStorageKeyWithEvaluation-8 320125 3753 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - StorageSlotKeyWithEvaluatedAddress(eval, bytes.Repeat([]byte{0xff}, 32)) - } -} diff --git a/trie/verkle.go b/trie/verkle.go deleted file mode 100644 index 186ac1f642..0000000000 --- a/trie/verkle.go +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb/database" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -var ( - errInvalidRootType = errors.New("invalid node type for root") -) - -// VerkleTrie is a wrapper around VerkleNode that implements the trie.Trie -// interface so that Verkle trees can be reused verbatim. -type VerkleTrie struct { - root verkle.VerkleNode - cache *utils.PointCache - reader *Reader - tracer *PrevalueTracer -} - -// NewVerkleTrie constructs a verkle tree based on the specified root hash. -func NewVerkleTrie(root common.Hash, db database.NodeDatabase, cache *utils.PointCache) (*VerkleTrie, error) { - reader, err := NewReader(root, common.Hash{}, db) - if err != nil { - return nil, err - } - t := &VerkleTrie{ - root: verkle.New(), - cache: cache, - reader: reader, - tracer: NewPrevalueTracer(), - } - // Parse the root verkle node if it's not empty. - if root != types.EmptyVerkleHash && root != types.EmptyRootHash { - blob, err := t.nodeResolver(nil) - if err != nil { - return nil, err - } - node, err := verkle.ParseNode(blob, 0) - if err != nil { - return nil, err - } - t.root = node - } - return t, nil -} - -// GetKey returns the sha3 preimage of a hashed key that was previously used -// to store a value. -func (t *VerkleTrie) GetKey(key []byte) []byte { - return key -} - -// GetAccount implements state.Trie, retrieving the account with the specified -// account address. If the specified account is not in the verkle tree, nil will -// be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error) { - var ( - acc = &types.StateAccount{} - values [][]byte - err error - ) - switch n := t.root.(type) { - case *verkle.InternalNode: - values, err = n.GetValuesAtStem(t.cache.GetStem(addr[:]), t.nodeResolver) - if err != nil { - return nil, fmt.Errorf("GetAccount (%x) error: %v", addr, err) - } - default: - return nil, errInvalidRootType - } - if values == nil { - return nil, nil - } - basicData := values[utils.BasicDataLeafKey] - acc.Nonce = binary.BigEndian.Uint64(basicData[utils.BasicDataNonceOffset:]) - acc.Balance = new(uint256.Int).SetBytes(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - acc.CodeHash = values[utils.CodeHashLeafKey] - - // TODO account.Root is leave as empty. How should we handle the legacy account? - return acc, nil -} - -// PrefetchAccount attempts to resolve specific accounts from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchAccount(addresses []common.Address) error { - for _, addr := range addresses { - if _, err := t.GetAccount(addr); err != nil { - return err - } - } - return nil -} - -// GetStorage implements state.Trie, retrieving the storage slot with the specified -// account address and storage key. If the specified slot is not in the verkle tree, -// nil will be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - val, err := t.root.Get(k, t.nodeResolver) - if err != nil { - return nil, err - } - return common.TrimLeftZeroes(val), nil -} - -// PrefetchStorage attempts to resolve specific storage slots from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { - for _, key := range keys { - if _, err := t.GetStorage(addr, key); err != nil { - return err - } - } - return nil -} - -// UpdateAccount implements state.Trie, writing the provided account into the tree. -// If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { - var ( - err error - basicData [32]byte - values = make([][]byte, verkle.NodeWidth) - stem = t.cache.GetStem(addr[:]) - ) - - // Code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. PutUint32(...) requires - // 4 bytes, so we need to shift the offset 1 byte to the left. - binary.BigEndian.PutUint32(basicData[utils.BasicDataCodeSizeOffset-1:], uint32(codeLen)) - binary.BigEndian.PutUint64(basicData[utils.BasicDataNonceOffset:], acc.Nonce) - if acc.Balance.ByteLen() > 16 { - panic("balance too large") - } - acc.Balance.WriteToSlice(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - values[utils.BasicDataLeafKey] = basicData[:] - values[utils.CodeHashLeafKey] = acc.CodeHash[:] - - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(stem, values, t.nodeResolver) - default: - return errInvalidRootType - } - if err != nil { - return fmt.Errorf("UpdateAccount (%x) error: %v", addr, err) - } - - return nil -} - -// UpdateStorage implements state.Trie, writing the provided storage slot into -// the tree. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateStorage(address common.Address, key, value []byte) error { - // Left padding the slot value to 32 bytes. - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) - } else { - copy(v[32-len(value):], value[:]) - } - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(address.Bytes()), key) - return t.root.Insert(k, v[:], t.nodeResolver) -} - -// DeleteAccount leaves the account untouched, as no account deletion can happen -// in verkle. -// There is a special corner case, in which an account that is prefunded, CREATE2-d -// and then SELFDESTRUCT-d should see its funds drained. EIP161 says that account -// should be removed, but this is verboten by the verkle spec. This contains a -// workaround in which the method checks for this corner case, and if so, overwrites -// the balance with 0. This will be removed once the spec has been clarified. -func (t *VerkleTrie) DeleteAccount(addr common.Address) error { - k := utils.BasicDataKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes())) - values, err := t.root.(*verkle.InternalNode).GetValuesAtStem(k, t.nodeResolver) - if err != nil { - return fmt.Errorf("Error getting data at %x in delete: %w", k, err) - } - var prefunded bool - for i, v := range values { - switch i { - case 0: - prefunded = len(v) == 32 - case 1: - prefunded = len(v) == 32 && bytes.Equal(v, types.EmptyCodeHash[:]) - default: - prefunded = v == nil - } - if !prefunded { - break - } - } - if prefunded { - t.root.Insert(k, common.Hash{}.Bytes(), t.nodeResolver) - } - return nil -} - -// RollBackAccount removes the account info + code from the tree, unlike DeleteAccount -// that will overwrite it with 0s. The first 64 storage slots are also removed. -func (t *VerkleTrie) RollBackAccount(addr common.Address) error { - var ( - evaluatedAddr = t.cache.Get(addr.Bytes()) - basicDataKey = utils.BasicDataKeyWithEvaluatedAddress(evaluatedAddr) - ) - basicDataBytes, err := t.root.Get(basicDataKey, t.nodeResolver) - if err != nil { - return fmt.Errorf("rollback: error finding code size: %w", err) - } - if len(basicDataBytes) == 0 { - return errors.New("rollback: basic data is not existent") - } - // The code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. - // LittleEndian.Uint32(...) expects 4-bytes, so we need to shift the offset 1-byte to the left. - codeSize := binary.BigEndian.Uint32(basicDataBytes[utils.BasicDataCodeSizeOffset-1:]) - - // Delete the account header + first 64 slots + first 128 code chunks - _, err = t.root.(*verkle.InternalNode).DeleteAtStem(basicDataKey[:31], t.nodeResolver) - if err != nil { - return fmt.Errorf("error rolling back account header: %w", err) - } - - // Delete all further code - for i, chunknr := uint64(31*128), uint64(128); i < uint64(codeSize); i, chunknr = i+31*256, chunknr+256 { - // evaluate group key at the start of a new group - offset := uint256.NewInt(chunknr) - key := utils.CodeChunkKeyWithEvaluatedAddress(evaluatedAddr, offset) - - if _, err = t.root.(*verkle.InternalNode).DeleteAtStem(key[:], t.nodeResolver); err != nil { - return fmt.Errorf("error deleting code chunk stem (addr=%x, offset=%d) error: %w", addr[:], offset, err) - } - } - return nil -} - -// DeleteStorage implements state.Trie, deleting the specified storage slot from -// the trie. If the storage slot was not existent in the trie, no error will be -// returned. If the trie is corrupted, an error will be returned. -func (t *VerkleTrie) DeleteStorage(addr common.Address, key []byte) error { - var zero [32]byte - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - return t.root.Insert(k, zero[:], t.nodeResolver) -} - -// Hash returns the root hash of the tree. It does not write to the database and -// can be used even if the tree doesn't have one. -func (t *VerkleTrie) Hash() common.Hash { - return t.root.Commit().Bytes() -} - -// Commit writes all nodes to the tree's memory database. -func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*verkle.InternalNode) - nodes, err := root.BatchSerialize() - if err != nil { - // Error return from this function indicates error in the code logic - // of BatchSerialize, and we fail catastrophically if this is the case. - panic(fmt.Errorf("BatchSerialize failed: %v", err)) - } - nodeset := trienode.NewNodeSet(common.Hash{}) - for _, node := range nodes { - // Hash parameter is not used in pathdb - nodeset.AddNode(node.Path, trienode.NewNodeWithPrev(common.Hash{}, node.SerializedBytes, t.tracer.Get(node.Path))) - } - // Serialize root commitment form - return t.Hash(), nodeset -} - -// NodeIterator implements state.Trie, returning an iterator that returns -// nodes of the trie. Iteration starts at the key after the given start key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) NodeIterator(startKey []byte) (NodeIterator, error) { - panic("not implemented") -} - -// Prove implements state.Trie, constructing a Merkle proof for key. The result -// contains all encoded nodes on the path to the value at key. The value itself -// is also included in the last node and can be retrieved by verifying the proof. -// -// If the trie does not contain a value for key, the returned proof contains all -// nodes of the longest existing prefix of the key (at least the root), ending -// with the node that proves the absence of the key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { - panic("not implemented") -} - -// Copy returns a deep-copied verkle tree. -func (t *VerkleTrie) Copy() *VerkleTrie { - return &VerkleTrie{ - root: t.root.Copy(), - cache: t.cache, - reader: t.reader, - tracer: t.tracer.Copy(), - } -} - -// IsVerkle indicates if the trie is a Verkle trie. -func (t *VerkleTrie) IsVerkle() bool { - return true -} - -// Proof builds and returns the verkle multiproof for keys, built against -// the pre tree. The post tree is passed in order to add the post values -// to that proof. -func (t *VerkleTrie) Proof(posttrie *VerkleTrie, keys [][]byte) (*verkle.VerkleProof, verkle.StateDiff, error) { - var postroot verkle.VerkleNode - if posttrie != nil { - postroot = posttrie.root - } - proof, _, _, _, err := verkle.MakeVerkleMultiProof(t.root, postroot, keys, t.nodeResolver) - if err != nil { - return nil, nil, err - } - p, kvps, err := verkle.SerializeProof(proof) - if err != nil { - return nil, nil, err - } - return p, kvps, nil -} - -// ChunkedCode represents a sequence of 32-bytes chunks of code (31 bytes of which -// are actual code, and 1 byte is the pushdata offset). -type ChunkedCode []byte - -// Copy the values here so as to avoid an import cycle -const ( - PUSH1 = byte(0x60) - PUSH32 = byte(0x7f) -) - -// ChunkifyCode generates the chunked version of an array representing EVM bytecode -func ChunkifyCode(code []byte) ChunkedCode { - var ( - chunkOffset = 0 // offset in the chunk - chunkCount = len(code) / 31 - codeOffset = 0 // offset in the code - ) - if len(code)%31 != 0 { - chunkCount++ - } - chunks := make([]byte, chunkCount*32) - for i := 0; i < chunkCount; i++ { - // number of bytes to copy, 31 unless the end of the code has been reached. - end := 31 * (i + 1) - if len(code) < end { - end = len(code) - } - copy(chunks[i*32+1:], code[31*i:end]) // copy the code itself - - // chunk offset = taken from the last chunk. - if chunkOffset > 31 { - // skip offset calculation if push data covers the whole chunk - chunks[i*32] = 31 - chunkOffset = 1 - continue - } - chunks[32*i] = byte(chunkOffset) - chunkOffset = 0 - - // Check each instruction and update the offset it should be 0 unless - // a PUSH-N overflows. - for ; codeOffset < end; codeOffset++ { - if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { - codeOffset += int(code[codeOffset] - PUSH1 + 1) - if codeOffset+1 >= 31*(i+1) { - codeOffset++ - chunkOffset = codeOffset - 31*(i+1) - break - } - } - } - } - return chunks -} - -// UpdateContractCode implements state.Trie, writing the provided contract code -// into the trie. -// Note that the code-size *must* be already saved by a previous UpdateAccount call. -func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { - var ( - chunks = ChunkifyCode(code) - values [][]byte - key []byte - err error - ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 - if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, verkle.NodeWidth) - key = utils.CodeChunkKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), uint256.NewInt(chunknr)) - } - values[groupOffset] = chunks[i : i+32] - - if groupOffset == 255 || len(chunks)-i <= 32 { - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(key[:31], values, t.nodeResolver) - if err != nil { - return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) - } - default: - return errInvalidRootType - } - } - } - return nil -} - -func (t *VerkleTrie) ToDot() string { - return verkle.ToDot(t.root) -} - -func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) { - blob, err := t.reader.Node(path, common.Hash{}) - if err != nil { - return nil, err - } - t.tracer.Put(path, blob) - return blob, nil -} - -// Witness returns a set containing all trie nodes that have been accessed. -func (t *VerkleTrie) Witness() map[string][]byte { - panic("not implemented") -} diff --git a/trie/verkle_test.go b/trie/verkle_test.go deleted file mode 100644 index 1832e3db13..0000000000 --- a/trie/verkle_test.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/holiman/uint256" -) - -var ( - accounts = map[common.Address]*types.StateAccount{ - {1}: { - Nonce: 100, - Balance: uint256.NewInt(100), - CodeHash: common.Hash{0x1}.Bytes(), - }, - {2}: { - Nonce: 200, - Balance: uint256.NewInt(200), - CodeHash: common.Hash{0x2}.Bytes(), - }, - } - storages = map[common.Address]map[common.Hash][]byte{ - {1}: { - common.Hash{10}: []byte{10}, - common.Hash{11}: []byte{11}, - common.MaxHash: []byte{0xff}, - }, - {2}: { - common.Hash{20}: []byte{20}, - common.Hash{21}: []byte{21}, - common.MaxHash: []byte{0xff}, - }, - } -) - -func TestVerkleTreeReadWrite(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - if err := tr.UpdateAccount(addr, acct, 0); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - } - - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } -} - -func TestVerkleRollBack(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - // create more than 128 chunks of code - code := make([]byte, 129*32) - for i := 0; i < len(code); i += 2 { - code[i] = 0x60 - code[i+1] = byte(i % 256) - } - if err := tr.UpdateAccount(addr, acct, len(code)); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - hash := crypto.Keccak256Hash(code) - if err := tr.UpdateContractCode(addr, hash, code); err != nil { - t.Fatalf("Failed to update contract, %v", err) - } - } - - // Check that things were created - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } - - // ensure there is some code in the 2nd group of the 1st account - keyOf2ndGroup := utils.CodeChunkKeyWithEvaluatedAddress(tr.cache.Get(common.Address{1}.Bytes()), uint256.NewInt(128)) - chunk, err := tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) == 0 { - t.Fatal("account was not created ") - } - - // Rollback first account and check that it is gone - addr1 := common.Address{1} - err = tr.RollBackAccount(addr1) - if err != nil { - t.Fatalf("error rolling back address 1: %v", err) - } - - // ensure the account is gone - stored, err := tr.GetAccount(addr1) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if stored != nil { - t.Fatal("account was not deleted") - } - - // ensure that the last code chunk is also gone from the tree - chunk, err = tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) != 0 { - t.Fatal("account was not deleted") - } -} diff --git a/triedb/database.go b/triedb/database.go index 331cea81f1..996acf667c 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -129,8 +129,8 @@ func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, er return db.backend.StateReader(blockRoot) } -// HistoricReader constructs a reader for accessing the requested historic state. -func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { +// HistoricStateReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricStateReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { pdb, ok := db.backend.(*pathdb.Database) if !ok { return nil, errors.New("not supported") @@ -138,6 +138,15 @@ func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateRea return pdb.HistoricReader(root) } +// HistoricNodeReader constructs a reader for accessing the historical trie node. +func (db *Database) HistoricNodeReader(root common.Hash) (*pathdb.HistoricalNodeReader, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return nil, errors.New("not supported") + } + return pdb.HistoricNodeReader(root) +} + // Update performs a state transition by committing dirty nodes contained in the // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 138962110f..853e1090b3 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -132,7 +132,7 @@ func (b *buffer) size() uint64 { // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. -func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { +func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { if b.done != nil { panic("duplicated flush operation") } @@ -165,11 +165,9 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.A // // This step is crucial to guarantee that the corresponding state history remains // available for state rollback. - if freezer != nil { - if err := freezer.SyncAncient(); err != nil { - b.flushErr = err - return - } + if err := syncHistory(freezers...); err != nil { + b.flushErr = err + return } nodes := b.nodes.write(batch, nodesCache) accounts, slots := b.states.write(batch, progress, statesCache) diff --git a/triedb/pathdb/config.go b/triedb/pathdb/config.go index 2675b701e3..7c29aece61 100644 --- a/triedb/pathdb/config.go +++ b/triedb/pathdb/config.go @@ -44,6 +44,26 @@ const ( // pause time will increase when the database writes happen. defaultBufferSize = 64 * 1024 * 1024 + // maxFullValueCheckpoint defines the maximum allowed encoding frequency (1/16) + // for storing nodes in full format. With this setting, a node may be written + // to the trienode history as a full value at the specified frequency. + // + // Note that the frequency is not strict: the actual decision is probabilistic. + // Only the overall long-term full-value encoding rate is enforced. + // + // Values beyond this limit are considered ineffective, as the trienode history + // is already well compressed. Increasing it further will only degrade read + // performance linearly. + maxFullValueCheckpoint = 16 + + // defaultFullValueCheckpoint defines the default full-value encoding frequency + // (1/8) for storing nodes in full format. With this setting, nodes may be + // written to the trienode history as full values at the specified rate. + // + // This strikes a balance between effective compression of the trienode history + // and acceptable read performance. + defaultFullValueCheckpoint = 8 + // Maximum diff layers allowed in the layer tree. DefaultMaxDiffLayers = 128 ) @@ -51,6 +71,8 @@ const ( // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ StateHistory: params.FullImmutabilityThreshold, + TrienodeHistory: -1, + FullValueCheckpoint: defaultFullValueCheckpoint, EnableStateIndexing: false, TrieCleanSize: defaultTrieCleanSize, StateCleanSize: defaultStateCleanSize, @@ -60,22 +82,28 @@ var Defaults = &Config{ // ReadOnly is the config in order to open database in read only mode. var ReadOnly = &Config{ - ReadOnly: true, - TrieCleanSize: defaultTrieCleanSize, - StateCleanSize: defaultStateCleanSize, - MaxDiffLayers: DefaultMaxDiffLayers, + ReadOnly: true, + TrienodeHistory: -1, + TrieCleanSize: defaultTrieCleanSize, + StateCleanSize: defaultStateCleanSize, + MaxDiffLayers: DefaultMaxDiffLayers, + FullValueCheckpoint: defaultFullValueCheckpoint, } // Config contains the settings for database. type Config struct { + TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data + StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data + WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer + ReadOnly bool // Flag whether the database is opened in read only mode + JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + + // Historical state configurations StateHistory uint64 // Number of recent blocks to maintain state history for, 0: full chain MaxDiffLayers int // Maximum diff layers allowed in the layer tree. + TrienodeHistory int64 // Number of recent blocks to maintain trienode history for, 0: full chain, negative: disable EnableStateIndexing bool // Whether to enable state history indexing for external state access - TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data - StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data - WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer - ReadOnly bool // Flag whether the database is opened in read only mode - JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + FullValueCheckpoint uint32 // The rate at which trie nodes are encoded in full-value format // Testing configurations SnapshotNoBuild bool // Flag Whether the state generation is disabled @@ -91,11 +119,18 @@ func (c *Config) sanitize() *Config { log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.WriteBufferSize), "updated", common.StorageSize(maxBufferSize)) conf.WriteBufferSize = maxBufferSize } - if conf.MaxDiffLayers <= 0 { log.Warn("Sanitizing invalid max diff layers", "provided", conf.MaxDiffLayers, "updated", DefaultMaxDiffLayers) conf.MaxDiffLayers = DefaultMaxDiffLayers } + if conf.FullValueCheckpoint > maxFullValueCheckpoint { + log.Warn("Sanitizing trienode history full value checkpoint", "provided", conf.FullValueCheckpoint, "updated", maxFullValueCheckpoint) + conf.FullValueCheckpoint = maxFullValueCheckpoint + } + if conf.FullValueCheckpoint == 0 { + conf.FullValueCheckpoint = 1 + log.Info("Disabling diff mode trie node history encoding") + } return &conf } @@ -114,6 +149,13 @@ func (c *Config) fields() []interface{} { } else { list = append(list, "state-history", fmt.Sprintf("last %d blocks", c.StateHistory)) } + if c.TrienodeHistory >= 0 { + if c.TrienodeHistory == 0 { + list = append(list, "trie-history", "entire chain") + } else { + list = append(list, "trie-history", fmt.Sprintf("last %d blocks", c.TrienodeHistory)) + } + } if c.EnableStateIndexing { list = append(list, "index-history", true) } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 0e00318c5a..2b4dff4c8a 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -31,8 +31,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-verkle" ) // layer is the interface implemented by all state layers which includes some @@ -45,7 +45,7 @@ type layer interface { // Note: // - the returned node is not a copy, please don't modify it. // - no error will be returned if the requested node is not found in database. - node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) + node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) // account directly retrieves the account RLP associated with a particular // hash in the slim data format. An error will be returned if the read @@ -97,16 +97,16 @@ func merkleNodeHasher(blob []byte) (common.Hash, error) { return crypto.Keccak256Hash(blob), nil } -// verkleNodeHasher computes the hash of the given verkle node. -func verkleNodeHasher(blob []byte) (common.Hash, error) { +// binaryNodeHasher computes the hash of the given verkle node. +func binaryNodeHasher(blob []byte) (common.Hash, error) { if len(blob) == 0 { return types.EmptyVerkleHash, nil } - n, err := verkle.ParseNode(blob, 0) + n, err := bintrie.DeserializeNode(blob, 0) if err != nil { return common.Hash{}, err } - return n.Commit().Bytes(), nil + return n.Hash(), nil } // Database is a multiple-layered structure for maintaining in-memory states @@ -137,6 +137,9 @@ type Database struct { stateFreezer ethdb.ResettableAncientStore // Freezer for storing state histories, nil possible in tests stateIndexer *historyIndexer // History indexer historical state data, nil possible + trienodeFreezer ethdb.ResettableAncientStore // Freezer for storing trienode histories, nil possible in tests + trienodeIndexer *historyIndexer // History indexer for historical trienode data + lock sync.RWMutex // Lock to prevent mutations from happening at the same time } @@ -163,17 +166,20 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { // compress the shared key prefix. if isVerkle { db.diskdb = rawdb.NewTable(diskdb, string(rawdb.VerklePrefix)) - db.hasher = verkleNodeHasher + db.hasher = binaryNodeHasher } // Construct the layer tree by resolving the in-disk singleton state // and in-memory layer journal. db.tree = newLayerTree(db.loadLayers()) - // Repair the state history, which might not be aligned with the state - // in the key-value store due to an unclean shutdown. - if err := db.repairHistory(); err != nil { - log.Crit("Failed to repair state history", "err", err) + // Repair the history, which might not be aligned with the persistent + // state in the key-value store due to an unclean shutdown. + states, trienodes, err := repairHistory(db.diskdb, isVerkle, db.config.ReadOnly, db.tree.bottom().stateID(), db.config.TrienodeHistory >= 0) + if err != nil { + log.Crit("Failed to repair history", "err", err) } + db.stateFreezer, db.trienodeFreezer = states, trienodes + // Disable database in case node is still in the initial state sync stage. if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly { if err := db.Disable(); err != nil { @@ -187,11 +193,8 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { if err := db.setStateGenerator(); err != nil { log.Crit("Failed to setup the generator", "err", err) } - // TODO (rjl493456442) disable the background indexing in read-only mode - if db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Enabled state history indexing") - } + db.setHistoryIndexer() + fields := config.fields() if db.isVerkle { fields = append(fields, "verkle", true) @@ -200,59 +203,28 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { return db } -// repairHistory truncates leftover state history objects, which may occur due -// to an unclean shutdown or other unexpected reasons. -func (db *Database) repairHistory() error { - // Open the freezer for state history. This mechanism ensures that - // only one database instance can be opened at a time to prevent - // accidental mutation. - ancient, err := db.diskdb.AncientDatadir() - if err != nil { - // TODO error out if ancient store is disabled. A tons of unit tests - // disable the ancient store thus the error here will immediately fail - // all of them. Fix the tests first. - return nil - } - freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly) - if err != nil { - log.Crit("Failed to open state history freezer", "err", err) +// setHistoryIndexer initializes the indexers for both state history and +// trienode history if available. Note that this function may be called while +// existing indexers are still running, so they must be closed beforehand. +func (db *Database) setHistoryIndexer() { + // TODO (rjl493456442) disable the background indexing in read-only mode + if !db.config.EnableStateIndexing { + return } - db.stateFreezer = freezer - - // Reset the entire state histories if the trie database is not initialized - // yet. This action is necessary because these state histories are not - // expected to exist without an initialized trie database. - id := db.tree.bottom().stateID() - if id == 0 { - frozen, err := db.stateFreezer.Ancients() - if err != nil { - log.Crit("Failed to retrieve head of state history", "err", err) - } - if frozen != 0 { - // Purge all state history indexing data first - batch := db.diskdb.NewBatch() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - log.Crit("Failed to purge state history index", "err", err) - } - if err := db.stateFreezer.Reset(); err != nil { - log.Crit("Failed to reset state histories", "err", err) - } - log.Info("Truncated extraneous state history") + if db.stateFreezer != nil { + if db.stateIndexer != nil { + db.stateIndexer.close() } - return nil - } - // Truncate the extra state histories above in freezer in case it's not - // aligned with the disk layer. It might happen after a unclean shutdown. - pruned, err := truncateFromHead(db.stateFreezer, typeStateHistory, id) - if err != nil { - log.Crit("Failed to truncate extra state histories", "err", err) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) + log.Info("Enabled state history indexing") } - if pruned != 0 { - log.Warn("Truncated extra state histories", "number", pruned) + if db.trienodeFreezer != nil { + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } + db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory) + log.Info("Enabled trienode history indexing") } - return nil } // setStateGenerator loads the state generation progress marker and potentially @@ -333,8 +305,13 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6 if err := db.modifyAllowed(); err != nil { return err } - // TODO(rjl493456442) tracking the origins in the following PRs. - if err := db.tree.add(root, parentRoot, block, NewNodeSetWithOrigin(nodes.Nodes(), nil), states); err != nil { + var nodesWithOrigins *nodeSetWithOrigin + if db.config.TrienodeHistory >= 0 { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.NodeAndOrigins()) + } else { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.Nodes(), nil) + } + if err := db.tree.add(root, parentRoot, block, nodesWithOrigins, states); err != nil { return err } // Keep 128 diff layers in the memory, persistent layer is 129th. @@ -422,18 +399,9 @@ func (db *Database) Enable(root common.Hash) error { // all root->id mappings should be removed as well. Since // mappings can be huge and might take a while to clear // them, just leave them in disk and wait for overwriting. - if db.stateFreezer != nil { - // Purge all state history indexing data first - batch.Reset() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - return err - } - if err := db.stateFreezer.Reset(); err != nil { - return err - } - } + purgeHistory(db.stateFreezer, db.diskdb, typeStateHistory) + purgeHistory(db.trienodeFreezer, db.diskdb, typeTrienodeHistory) + // Re-enable the database as the final step. db.waitSync = false rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished) @@ -446,11 +414,8 @@ func (db *Database) Enable(root common.Hash) error { // To ensure the history indexer always matches the current state, we must: // 1. Close any existing indexer // 2. Re-initialize the indexer so it starts indexing from the new state root. - if db.stateIndexer != nil && db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer.close() - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Re-enabled state history indexing") - } + db.setHistoryIndexer() + log.Info("Rebuilt trie database", "root", root) return nil } @@ -506,6 +471,12 @@ func (db *Database) Recover(root common.Hash) error { if err != nil { return err } + if db.trienodeFreezer != nil { + _, err = truncateFromHead(db.trienodeFreezer, typeTrienodeHistory, dl.stateID()) + if err != nil { + return err + } + } log.Debug("Recovered state", "root", root, "elapsed", common.PrettyDuration(time.Since(start))) return nil } @@ -566,11 +537,21 @@ func (db *Database) Close() error { if db.stateIndexer != nil { db.stateIndexer.close() } + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } // Close the attached state history freezer. - if db.stateFreezer == nil { - return nil + if db.stateFreezer != nil { + if err := db.stateFreezer.Close(); err != nil { + return err + } + } + if db.trienodeFreezer != nil { + if err := db.trienodeFreezer.Close(); err != nil { + return err + } } - return db.stateFreezer.Close() + return nil } // Size returns the current storage size of the memory cache in front of the diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 31fcac3c84..844877ad6a 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -900,7 +900,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { var ( dIndex int roots = env.roots - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dRoot { @@ -961,7 +961,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { // Ensure the truncated state histories become accessible bRoot = env.db.tree.bottom().rootHash() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) for i, root := range roots { if root == bRoot { break diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index ae523c979c..8ca3a39cf3 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -79,7 +79,7 @@ func (dl *diffLayer) parentLayer() layer { // node implements the layer interface, retrieving the trie node blob with the // provided node information. No error will be returned if the node is not found. -func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { // Hold the lock, ensure the parent won't be changed during the // state accessing. dl.lock.RLock() @@ -91,7 +91,7 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeHitDepthHist.Update(int64(depth)) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) - return n.Blob, n.Hash, &nodeLoc{loc: locDiffLayer, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDiffLayer, depth: depth}, nil } // Trie node unknown to this layer, resolve from parent return dl.parent.node(owner, path, depth+1) diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 42cb703d1b..ed8027ae5f 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -111,12 +112,12 @@ func (dl *diskLayer) markStale() { // node implements the layer interface, retrieving the trie node with the // provided node info. No error will be returned if the node is not found. -func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { dl.lock.RLock() defer dl.lock.RUnlock() if dl.stale { - return nil, common.Hash{}, nil, errSnapshotStale + return nil, common.Hash{}, nodeLoc{}, errSnapshotStale } // Try to retrieve the trie node from the not-yet-written node buffer first // (both the live one and the frozen one). Note the buffer is lock free since @@ -128,7 +129,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) dirtyNodeHitDepthHist.Update(int64(depth)) - return n.Blob, n.Hash, &nodeLoc{loc: locDirtyCache, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDirtyCache, depth: depth}, nil } } } @@ -140,7 +141,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co if blob := dl.nodes.Get(nil, key); len(blob) > 0 { cleanNodeHitMeter.Mark(1) cleanNodeReadMeter.Mark(int64(len(blob))) - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locCleanCache, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locCleanCache, depth: depth}, nil } cleanNodeMissMeter.Mark(1) } @@ -160,7 +161,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dl.nodes.Set(key, blob) cleanNodeWriteMeter.Mark(int64(len(blob))) } - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locDiskLayer, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locDiskLayer, depth: depth}, nil } // account directly retrieves the account RLP associated with a particular @@ -278,7 +279,7 @@ func (dl *diskLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // If the layer is being generated, ensure the requested storage slot // has already been covered by the generator. - key := append(accountHash[:], storageHash[:]...) + key := storageKeySlice(accountHash, storageHash) marker := dl.genMarker() if marker != nil && bytes.Compare(key, marker) > 0 { return nil, errNotCoveredYet @@ -329,36 +330,61 @@ func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes *no return newDiffLayer(dl, root, id, block, nodes, states) } -// writeStateHistory stores the state history and indexes if indexing is +// writeHistory stores the specified history and indexes if indexing is // permitted. // // What's more, this function also returns a flag indicating whether the // buffer flushing is required, ensuring the persistent state ID is always // greater than or equal to the first history ID. -func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { - // Short circuit if state history is not permitted - if dl.db.stateFreezer == nil { +func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error) { + var ( + limit uint64 + freezer ethdb.AncientStore + indexer *historyIndexer + writeFunc func(writer ethdb.AncientWriter, dl *diffLayer) error + ) + switch typ { + case typeStateHistory: + freezer = dl.db.stateFreezer + indexer = dl.db.stateIndexer + writeFunc = writeStateHistory + limit = dl.db.config.StateHistory + case typeTrienodeHistory: + freezer = dl.db.trienodeFreezer + indexer = dl.db.trienodeIndexer + writeFunc = func(writer ethdb.AncientWriter, diff *diffLayer) error { + return writeTrienodeHistory(writer, diff, dl.db.config.FullValueCheckpoint) + } + // Skip the history commit if the trienode history is not permitted + if dl.db.config.TrienodeHistory < 0 { + return false, nil + } + limit = uint64(dl.db.config.TrienodeHistory) + default: + panic(fmt.Sprintf("unknown history type: %v", typ)) + } + // Short circuit if the history freezer is nil + if freezer == nil { return false, nil } // Bail out with an error if writing the state history fails. // This can happen, for example, if the device is full. - err := writeStateHistory(dl.db.stateFreezer, diff) + err := writeFunc(freezer, diff) if err != nil { return false, err } - // Notify the state history indexer for newly created history - if dl.db.stateIndexer != nil { - if err := dl.db.stateIndexer.extend(diff.stateID()); err != nil { + // Notify the history indexer for newly created history + if indexer != nil { + if err := indexer.extend(diff.stateID()); err != nil { return false, err } } // Determine if the persisted history object has exceeded the // configured limitation. - limit := dl.db.config.StateHistory if limit == 0 { return false, nil } - tail, err := dl.db.stateFreezer.Tail() + tail, err := freezer.Tail() if err != nil { return false, err } // firstID = tail+1 @@ -381,14 +407,14 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { // These measures ensure the persisted state ID always remains greater // than or equal to the first history ID. if persistentID := rawdb.ReadPersistentStateID(dl.db.diskdb); persistentID < newFirst { - log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) + log.Debug("Skip tail truncation", "type", typ, "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) return true, nil } - pruned, err := truncateFromTail(dl.db.stateFreezer, typeStateHistory, newFirst-1) + pruned, err := truncateFromTail(freezer, typ, newFirst-1) if err != nil { return false, err } - log.Debug("Pruned state history", "items", pruned, "tailid", newFirst) + log.Debug("Pruned history", "type", typ, "items", pruned, "tailid", newFirst) return false, nil } @@ -402,10 +428,22 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Construct and store the state history first. If crash happens after storing // the state history but without flushing the corresponding states(journal), // the stored state history will be truncated from head in the next restart. - flush, err := dl.writeStateHistory(bottom) + flushA, err := dl.writeHistory(typeStateHistory, bottom) if err != nil { return nil, err } + // Construct and store the trienode history first. If crash happens after + // storing the trienode history but without flushing the corresponding + // states(journal), the stored trienode history will be truncated from head + // in the next restart. + flushB, err := dl.writeHistory(typeTrienodeHistory, bottom) + if err != nil { + return nil, err + } + // Since the state history and trienode history may be configured with different + // lengths, the buffer will be flushed once either of them meets its threshold. + flush := flushA || flushB + // Mark the diskLayer as stale before applying any mutations on top. dl.stale = true @@ -454,7 +492,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Freeze the live buffer and schedule background flushing dl.frozen = combined - dl.frozen.flush(bottom.root, dl.db.diskdb, dl.db.stateFreezer, progress, dl.nodes, dl.states, bottom.stateID(), func() { + dl.frozen.flush(bottom.root, dl.db.diskdb, []ethdb.AncientWriter{dl.db.stateFreezer, dl.db.trienodeFreezer}, progress, dl.nodes, dl.states, bottom.stateID(), func() { // Resume the background generation if it's not completed yet. // The generator is assumed to be available if the progress is // not nil. @@ -510,12 +548,17 @@ func (dl *diskLayer) revert(h *stateHistory) (*diskLayer, error) { dl.stale = true - // Unindex the corresponding state history + // Unindex the corresponding history if dl.db.stateIndexer != nil { if err := dl.db.stateIndexer.shorten(dl.id); err != nil { return nil, err } } + if dl.db.trienodeIndexer != nil { + if err := dl.db.trienodeIndexer.shorten(dl.id); err != nil { + return nil, err + } + } // State change may be applied to node buffer, or the persistent // state, depends on if node buffer is empty or not. If the node // buffer is not empty, it means that the state transition that diff --git a/triedb/pathdb/flush.go b/triedb/pathdb/flush.go index 6563dbccff..4f816cf6a6 100644 --- a/triedb/pathdb/flush.go +++ b/triedb/pathdb/flush.go @@ -116,15 +116,16 @@ func writeStates(batch ethdb.Batch, genMarker []byte, accountData map[common.Has continue } slots += 1 + key := storageKeySlice(addrHash, storageHash) if len(blob) == 0 { rawdb.DeleteStorageSnapshot(batch, addrHash, storageHash) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), nil) + clean.Set(key, nil) } } else { rawdb.WriteStorageSnapshot(batch, addrHash, storageHash, blob) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), blob) + clean.Set(key, blob) } } } diff --git a/triedb/pathdb/generate.go b/triedb/pathdb/generate.go index 2efbbbb4e1..d3d26fff26 100644 --- a/triedb/pathdb/generate.go +++ b/triedb/pathdb/generate.go @@ -148,6 +148,7 @@ func (g *generator) stop() { g.abort <- ch <-ch g.running = false + log.Debug("Snapshot generation has been terminated") } // completed returns the flag indicating if the whole generation is done. diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index d78999f218..820c3c03bf 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -22,6 +22,7 @@ import ( "iter" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -121,6 +122,20 @@ func (ident stateIdent) String() string { return ident.addressHash.Hex() + ident.path } +func (ident stateIdent) bloomSize() int { + if ident.typ == typeAccount { + return 0 + } + if ident.typ == typeStorage { + return 0 + } + scheme := accountIndexScheme + if ident.addressHash != (common.Hash{}) { + scheme = storageIndexScheme + } + return scheme.getBitmapSize(len(ident.path)) +} + // newAccountIdent constructs a state identifier for an account. func newAccountIdent(addressHash common.Hash) stateIdent { return stateIdent{ @@ -180,17 +195,62 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora } } -// newTrienodeIdentQuery constructs a state identifier for a trie node. -// the addressHash denotes the address hash of the associated account; -// the path denotes the path of the node within the trie; -// -// nolint:unused -func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery { - return stateIdentQuery{ - stateIdent: newTrienodeIdent(addrHash, string(path)), +// indexElem defines the element for indexing. +type indexElem interface { + key() stateIdent + ext() []uint16 +} + +type accountIndexElem struct { + addressHash common.Hash +} + +func (a accountIndexElem) key() stateIdent { + return stateIdent{ + typ: typeAccount, + addressHash: a.addressHash, } } +func (a accountIndexElem) ext() []uint16 { + return nil +} + +type storageIndexElem struct { + addressHash common.Hash + storageHash common.Hash +} + +func (a storageIndexElem) key() stateIdent { + return stateIdent{ + typ: typeStorage, + addressHash: a.addressHash, + storageHash: a.storageHash, + } +} + +func (a storageIndexElem) ext() []uint16 { + return nil +} + +type trienodeIndexElem struct { + owner common.Hash + path string + data []uint16 +} + +func (a trienodeIndexElem) key() stateIdent { + return stateIdent{ + typ: typeTrienode, + addressHash: a.owner, + path: a.path, + } +} + +func (a trienodeIndexElem) ext() []uint16 { + return a.data +} + // history defines the interface of historical data, shared by stateHistory // and trienodeHistory. type history interface { @@ -198,7 +258,7 @@ type history interface { typ() historyType // forEach returns an iterator to traverse the state entries in the history. - forEach() iter.Seq[stateIdent] + forEach() iter.Seq[indexElem] } var ( @@ -262,3 +322,133 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) ( // Associated root->id mappings are left in the database. return int(ntail - otail), nil } + +// purgeHistory resets the history and also purges the associated index data. +func purgeHistory(store ethdb.ResettableAncientStore, disk ethdb.KeyValueStore, typ historyType) { + if store == nil { + return + } + frozen, err := store.Ancients() + if err != nil { + log.Crit("Failed to retrieve head of history", "type", typ, "err", err) + } + if frozen == 0 { + return + } + // Purge all state history indexing data first + batch := disk.NewBatch() + if typ == typeStateHistory { + rawdb.DeleteStateHistoryIndexMetadata(batch) + rawdb.DeleteStateHistoryIndexes(batch) + } else { + rawdb.DeleteTrienodeHistoryIndexMetadata(batch) + rawdb.DeleteTrienodeHistoryIndexes(batch) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to purge history index", "type", typ, "err", err) + } + if err := store.Reset(); err != nil { + log.Crit("Failed to reset history", "type", typ, "err", err) + } + log.Info("Truncated extraneous history", "type", typ) +} + +// syncHistory explicitly sync the provided history stores. +func syncHistory(stores ...ethdb.AncientWriter) error { + for _, store := range stores { + if store == nil { + continue + } + if err := store.SyncAncient(); err != nil { + return err + } + } + return nil +} + +// repairHistory truncates any leftover history objects in either the state +// history or the trienode history, which may occur due to an unclean shutdown +// or other unexpected events. +// +// Additionally, this mechanism ensures that the state history and trienode +// history remain aligned. Since the trienode history is optional and not +// required by regular users, a gap between the trienode history and the +// persistent state may appear if the trienode history was disabled during the +// previous run. This process detects and resolves such gaps, preventing +// unexpected panics. +func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint64, enableTrienode bool) (ethdb.ResettableAncientStore, ethdb.ResettableAncientStore, error) { + ancient, err := db.AncientDatadir() + if err != nil { + // TODO error out if ancient store is disabled. A tons of unit tests + // disable the ancient store thus the error here will immediately fail + // all of them. Fix the tests first. + return nil, nil, nil + } + // State history is mandatory as it is the key component that ensures + // resilience to deep reorgs. + states, err := rawdb.NewStateFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open state history freezer", "err", err) + } + + // Trienode history is optional and only required for building archive + // node with state proofs. + var trienodes ethdb.ResettableAncientStore + if enableTrienode { + trienodes, err = rawdb.NewTrienodeFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open trienode history freezer", "err", err) + } + } + + // Reset the both histories if the trie database is not initialized yet. + // This action is necessary because these histories are not expected + // to exist without an initialized trie database. + if stateID == 0 { + purgeHistory(states, db, typeStateHistory) + purgeHistory(trienodes, db, typeTrienodeHistory) + return states, trienodes, nil + } + // Truncate excessive history entries in either the state history or + // the trienode history, ensuring both histories remain aligned with + // the state. + head, err := states.Ancients() + if err != nil { + return nil, nil, err + } + if stateID > head { + return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, head) + } + if trienodes != nil { + th, err := trienodes.Ancients() + if err != nil { + return nil, nil, err + } + if stateID > th { + return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, th) + } + if th != head { + log.Info("Histories are not aligned with each other", "state", head, "trienode", th) + head = min(head, th) + } + } + head = min(head, stateID) + + // Truncate the extra history elements above in freezer in case it's not + // aligned with the state. It might happen after an unclean shutdown. + truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) { + if store == nil { + return + } + pruned, err := truncateFromHead(store, typ, nhead) + if err != nil { + log.Crit("Failed to truncate extra histories", "typ", typ, "err", err) + } + if pruned != 0 { + log.Warn("Truncated extra histories", "typ", typ, "number", pruned) + } + } + truncate(states, typeStateHistory, head) + truncate(trienodes, typeTrienodeHistory, head) + return states, trienodes, nil +} diff --git a/triedb/pathdb/history_index.go b/triedb/pathdb/history_index.go index 5b4c91d7e6..eee4c10273 100644 --- a/triedb/pathdb/history_index.go +++ b/triedb/pathdb/history_index.go @@ -20,28 +20,33 @@ import ( "errors" "fmt" "math" - "sort" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" ) -// parseIndex parses the index data with the supplied byte stream. The index data -// is a list of fixed-sized metadata. Empty metadata is regarded as invalid. -func parseIndex(blob []byte) ([]*indexBlockDesc, error) { +// parseIndex parses the index data from the provided byte stream. The index data +// is a sequence of fixed-size metadata entries, and any empty metadata entry is +// considered invalid. +// +// Each metadata entry consists of two components: the indexBlockDesc and an +// optional extension bitmap. The bitmap length may vary across different categories, +// but must remain consistent within the same category. +func parseIndex(blob []byte, bitmapSize int) ([]*indexBlockDesc, error) { if len(blob) == 0 { return nil, errors.New("empty state history index") } - if len(blob)%indexBlockDescSize != 0 { - return nil, fmt.Errorf("corrupted state index, len: %d", len(blob)) + size := indexBlockDescSize + bitmapSize + if len(blob)%size != 0 { + return nil, fmt.Errorf("corrupted state index, len: %d, bitmap size: %d", len(blob), bitmapSize) } var ( lastID uint32 - descList []*indexBlockDesc + descList = make([]*indexBlockDesc, 0, len(blob)/size) ) - for i := 0; i < len(blob)/indexBlockDescSize; i++ { + for i := 0; i < len(blob)/size; i++ { var desc indexBlockDesc - desc.decode(blob[i*indexBlockDescSize : (i+1)*indexBlockDescSize]) + desc.decode(blob[i*size : (i+1)*size]) if desc.empty() { return nil, errors.New("empty state history index block") } @@ -70,33 +75,35 @@ func parseIndex(blob []byte) ([]*indexBlockDesc, error) { // indexReader is the structure to look up the state history index records // associated with the specific state element. type indexReader struct { - db ethdb.KeyValueReader - descList []*indexBlockDesc - readers map[uint32]*blockReader - state stateIdent + db ethdb.KeyValueReader + descList []*indexBlockDesc + readers map[uint32]*blockReader + state stateIdent + bitmapSize int } // loadIndexData loads the index data associated with the specified state. -func loadIndexData(db ethdb.KeyValueReader, state stateIdent) ([]*indexBlockDesc, error) { +func loadIndexData(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) ([]*indexBlockDesc, error) { blob := readStateIndex(state, db) if len(blob) == 0 { return nil, nil } - return parseIndex(blob) + return parseIndex(blob, bitmapSize) } // newIndexReader constructs a index reader for the specified state. Reader with // empty data is allowed. -func newIndexReader(db ethdb.KeyValueReader, state stateIdent) (*indexReader, error) { - descList, err := loadIndexData(db, state) +func newIndexReader(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) (*indexReader, error) { + descList, err := loadIndexData(db, state, bitmapSize) if err != nil { return nil, err } return &indexReader{ - descList: descList, - readers: make(map[uint32]*blockReader), - db: db, - state: state, + descList: descList, + readers: make(map[uint32]*blockReader), + db: db, + state: state, + bitmapSize: bitmapSize, }, nil } @@ -107,11 +114,9 @@ func (r *indexReader) refresh() error { // may have been modified by additional elements written to the disk. if len(r.descList) != 0 { last := r.descList[len(r.descList)-1] - if !last.full() { - delete(r.readers, last.id) - } + delete(r.readers, last.id) } - descList, err := loadIndexData(r.db, r.state) + descList, err := loadIndexData(r.db, r.state, r.bitmapSize) if err != nil { return err } @@ -122,27 +127,15 @@ func (r *indexReader) refresh() error { // readGreaterThan locates the first element that is greater than the specified // id. If no such element is found, MaxUint64 is returned. func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { - index := sort.Search(len(r.descList), func(i int) bool { - return id < r.descList[i].max - }) - if index == len(r.descList) { - return math.MaxUint64, nil + it := r.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { + return 0, err } - desc := r.descList[index] - - br, ok := r.readers[desc.id] - if !ok { - var err error - blob := readStateIndexBlock(r.state, r.db, desc.id) - br, err = newBlockReader(blob) - if err != nil { - return 0, err - } - r.readers[desc.id] = br + if !found { + return math.MaxUint64, nil } - // The supplied ID is not greater than block.max, ensuring that an element - // satisfying the condition can be found. - return br.readGreaterThan(id) + return it.ID(), nil } // indexWriter is responsible for writing index data for a specific state (either @@ -152,57 +145,75 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { // history ids) is stored in these second-layer index blocks, which are size // limited. type indexWriter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - frozen []*blockWriter // The finalized index block writers, waiting for flush - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + frozen []*blockWriter // The finalized index block writers, waiting for flush + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } -// newIndexWriter constructs the index writer for the specified state. -func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, error) { +// newIndexWriter constructs the index writer for the specified state. Additionally, +// it takes an integer as the limit and prunes all existing elements above that ID. +// It's essential as the recovery mechanism after unclean shutdown during the history +// indexing. +func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexWriter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexWriter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the last block for appending new elements lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexWriter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } // append adds the new element into the index writer. -func (w *indexWriter) append(id uint64) error { +func (w *indexWriter) append(id uint64, ext []uint16) error { if id <= w.lastID { return fmt.Errorf("append element out of order, last: %d, this: %d", w.lastID, id) } - if w.bw.full() { + if w.bw.estimateFull(ext) { if err := w.rotate(); err != nil { return err } } - if err := w.bw.append(id); err != nil { + if err := w.bw.append(id, ext); err != nil { return err } w.lastID = id @@ -215,10 +226,10 @@ func (w *indexWriter) append(id uint64) error { func (w *indexWriter) rotate() error { var ( err error - desc = newIndexBlockDesc(w.bw.desc.id + 1) + desc = newIndexBlockDesc(w.bw.desc.id+1, w.bitmapSize) ) w.frozen = append(w.frozen, w.bw) - w.bw, err = newBlockWriter(nil, desc) + w.bw, err = newBlockWriter(nil, desc, 0 /* useless if the block is empty */, w.bitmapSize != 0) if err != nil { return err } @@ -250,7 +261,8 @@ func (w *indexWriter) finish(batch ethdb.Batch) { } w.frozen = nil // release all the frozen writers - buf := make([]byte, 0, indexBlockDescSize*len(descList)) + size := indexBlockDescSize + w.bitmapSize + buf := make([]byte, 0, size*len(descList)) for _, desc := range descList { buf = append(buf, desc.encode()...) } @@ -259,49 +271,64 @@ func (w *indexWriter) finish(batch ethdb.Batch) { // indexDeleter is responsible for deleting index data for a specific state. type indexDeleter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - dropped []uint32 // The list of index block id waiting for deleting - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + dropped []uint32 // The list of index block id waiting for deleting + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } // newIndexDeleter constructs the index deleter for the specified state. -func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, error) { +func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexDeleter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { // TODO(rjl493456442) we can probably return an error here, // deleter with no data is meaningless. - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexDeleter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the block for deleting element from lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexDeleter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } -// empty returns an flag indicating whether the state index is empty. +// empty returns whether the state index is empty. func (d *indexDeleter) empty() bool { return d.bw.empty() && len(d.descList) == 1 } @@ -334,7 +361,7 @@ func (d *indexDeleter) pop(id uint64) error { // Open the previous block writer for deleting lastDesc := d.descList[len(d.descList)-1] indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + bw, err := newBlockWriter(indexBlock, lastDesc, lastDesc.max, d.bitmapSize != 0) if err != nil { return err } @@ -360,7 +387,8 @@ func (d *indexDeleter) finish(batch ethdb.Batch) { if d.empty() { deleteStateIndex(d.state, batch) } else { - buf := make([]byte, 0, indexBlockDescSize*len(d.descList)) + size := indexBlockDescSize + d.bitmapSize + buf := make([]byte, 0, size*len(d.descList)) for _, desc := range d.descList { buf = append(buf, desc.encode()...) } diff --git a/triedb/pathdb/history_index_block.go b/triedb/pathdb/history_index_block.go index 5abdee682a..bb823bb13f 100644 --- a/triedb/pathdb/history_index_block.go +++ b/triedb/pathdb/history_index_block.go @@ -17,31 +17,37 @@ package pathdb import ( + "bytes" "encoding/binary" "errors" "fmt" "math" - "sort" + + "github.com/ethereum/go-ethereum/log" ) const ( - indexBlockDescSize = 14 // The size of index block descriptor - indexBlockEntriesCap = 4096 // The maximum number of entries can be grouped in a block - indexBlockRestartLen = 256 // The restart interval length of index block - historyIndexBatch = 512 * 1024 // The number of state history indexes for constructing or deleting as batch + indexBlockDescSize = 14 // The size of index block descriptor + indexBlockMaxSize = 4096 // The maximum size of a single index block + indexBlockRestartLen = 256 // The restart interval length of index block ) // indexBlockDesc represents a descriptor for an index block, which contains a // list of state mutation records associated with a specific state (either an // account or a storage slot). type indexBlockDesc struct { - max uint64 // The maximum state ID retained within the block - entries uint16 // The number of state mutation records retained within the block - id uint32 // The id of the index block + max uint64 // The maximum state ID retained within the block + entries uint16 // The number of state mutation records retained within the block + id uint32 // The id of the index block + extBitmap []byte // Optional fixed-size bitmap for the included extension elements } -func newIndexBlockDesc(id uint32) *indexBlockDesc { - return &indexBlockDesc{id: id} +func newIndexBlockDesc(id uint32, bitmapSize int) *indexBlockDesc { + var bitmap []byte + if bitmapSize > 0 { + bitmap = make([]byte, bitmapSize) + } + return &indexBlockDesc{id: id, extBitmap: bitmap} } // empty indicates whether the block is empty with no element retained. @@ -49,26 +55,33 @@ func (d *indexBlockDesc) empty() bool { return d.entries == 0 } -// full indicates whether the number of elements in the block exceeds the -// preconfigured limit. -func (d *indexBlockDesc) full() bool { - return d.entries >= indexBlockEntriesCap -} - // encode packs index block descriptor into byte stream. func (d *indexBlockDesc) encode() []byte { - var buf [indexBlockDescSize]byte + buf := make([]byte, indexBlockDescSize+len(d.extBitmap)) binary.BigEndian.PutUint64(buf[0:8], d.max) binary.BigEndian.PutUint16(buf[8:10], d.entries) binary.BigEndian.PutUint32(buf[10:14], d.id) + copy(buf[indexBlockDescSize:], d.extBitmap) return buf[:] } -// decode unpacks index block descriptor from byte stream. +// decode unpacks index block descriptor from byte stream. It's unsafe to mutate +// the provided byte stream after the function call. func (d *indexBlockDesc) decode(blob []byte) { d.max = binary.BigEndian.Uint64(blob[:8]) d.entries = binary.BigEndian.Uint16(blob[8:10]) d.id = binary.BigEndian.Uint32(blob[10:14]) + d.extBitmap = blob[indexBlockDescSize:] // no-deep copy! +} + +// copy returns a deep-copied object. +func (d *indexBlockDesc) copy() *indexBlockDesc { + return &indexBlockDesc{ + max: d.max, + entries: d.entries, + id: d.id, + extBitmap: bytes.Clone(d.extBitmap), + } } // parseIndexBlock parses the index block with the supplied byte stream. @@ -96,20 +109,38 @@ func (d *indexBlockDesc) decode(blob []byte) { // A uint16 can cover offsets in the range [0, 65536), which is more than enough // to store 4096 integers. // -// Each chunk begins with the full value of the first integer, followed by -// subsequent integers representing the differences between the current value -// and the preceding one. Integers are encoded with variable-size for best -// storage efficiency. Each chunk can be illustrated as below. +// Each chunk begins with a full integer value for the first element, followed +// by subsequent integers encoded as differences (deltas) from their preceding +// values. All integers use variable-length encoding for optimal space efficiency. +// +// In the updated format, each element in the chunk may optionally include an +// "extension" section. If an extension is present, it starts with a var-size +// integer indicating the length of the remaining extension payload, followed by +// that many bytes. If no extension is present, the element format is identical +// to the original version (i.e., only the integer or delta value is encoded). // -// Restart ---> +----------------+ -// | Full integer | -// +----------------+ -// | Diff with prev | -// +----------------+ -// | ... | -// +----------------+ -// | Diff with prev | -// +----------------+ +// In the trienode history index, the extension field contains the list of +// trie node IDs that fall within this range. For the given state transition, +// these IDs represent the specific nodes in this range that were mutated. +// +// Whether an element includes an extension is determined by the block reader +// based on the specification. Conceptually, a chunk is structured as: +// +// Restart ---> +----------------+ +// | Full integer | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | ... | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ // // Empty index block is regarded as invalid. func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { @@ -147,104 +178,98 @@ func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { type blockReader struct { restarts []uint16 data []byte + hasExt bool } // newBlockReader constructs the block reader with the supplied block data. -func newBlockReader(blob []byte) (*blockReader, error) { +func newBlockReader(blob []byte, hasExt bool) (*blockReader, error) { restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } return &blockReader{ restarts: restarts, - data: data, // safe to own the slice + data: data, // safe to own the slice + hasExt: hasExt, // flag whether extension should be resolved }, nil } // readGreaterThan locates the first element in the block that is greater than // the specified value. If no such element is found, MaxUint64 is returned. func (br *blockReader) readGreaterThan(id uint64) (uint64, error) { - var err error - index := sort.Search(len(br.restarts), func(i int) bool { - item, n := binary.Uvarint(br.data[br.restarts[i]:]) - if n <= 0 { - err = fmt.Errorf("failed to decode item at restart %d", br.restarts[i]) - } - return item > id - }) - if err != nil { + it := br.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { return 0, err } - if index == 0 { - item, _ := binary.Uvarint(br.data[br.restarts[0]:]) - return item, nil - } - var ( - start int - limit int - result uint64 - ) - if index == len(br.restarts) { - // The element being searched falls within the last restart section, - // there is no guarantee such element can be found. - start = int(br.restarts[len(br.restarts)-1]) - limit = len(br.data) - } else { - // The element being searched falls within the non-last restart section, - // such element can be found for sure. - start = int(br.restarts[index-1]) - limit = int(br.restarts[index]) - } - pos := start - for pos < limit { - x, n := binary.Uvarint(br.data[pos:]) - if pos == start { - result = x - } else { - result += x - } - if result > id { - return result, nil - } - pos += n - } - // The element which is greater than specified id is not found. - if index == len(br.restarts) { + if !found { return math.MaxUint64, nil } - // The element which is the first one greater than the specified id - // is exactly the one located at the restart point. - item, _ := binary.Uvarint(br.data[br.restarts[index]:]) - return item, nil + return it.ID(), nil } type blockWriter struct { desc *indexBlockDesc // Descriptor of the block restarts []uint16 // Offsets into the data slice, marking the start of each section data []byte // Aggregated encoded data slice + hasExt bool // Flag whether the extension field for each element exists } -func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) { +// newBlockWriter constructs a block writer. In addition to the existing data +// and block description, it takes an element ID and prunes all existing elements +// above that ID. It's essential as the recovery mechanism after unclean shutdown +// during the history indexing. +func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64, hasExt bool) (*blockWriter, error) { if len(blob) == 0 { return &blockWriter{ - desc: desc, - data: make([]byte, 0, 1024), + desc: desc, + data: make([]byte, 0, 1024), + hasExt: hasExt, }, nil } restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } - return &blockWriter{ + writer := &blockWriter{ desc: desc, restarts: restarts, data: data, // safe to own the slice - }, nil + hasExt: hasExt, + } + var trimmed int + for !writer.empty() && writer.last() > limit { + if err := writer.pop(writer.last()); err != nil { + return nil, err + } + trimmed += 1 + } + if trimmed > 0 { + log.Debug("Truncated extraneous elements", "count", trimmed, "limit", limit) + } + return writer, nil +} + +// setBitmap applies the given extension elements into the bitmap. +func (b *blockWriter) setBitmap(ext []uint16) { + for _, n := range ext { + // Node ID zero is intentionally filtered out. Any element in this range + // can indicate that the sub-tree's root node was mutated, so storing zero + // is redundant and saves one byte for bitmap. + if n != 0 { + setBit(b.desc.extBitmap, int(n-1)) + } + } } // append adds a new element to the block. The new element must be greater than // the previous one. The provided ID is assumed to always be greater than 0. -func (b *blockWriter) append(id uint64) error { +// +// ext refers to the optional extension field attached to the appended element. +// This extension mechanism is used by trie-node history and represents a list of +// trie node IDs that fall within the range covered by the index element +// (typically corresponding to a sub-trie in trie-node history). +func (b *blockWriter) append(id uint64, ext []uint16) error { if id == 0 { return errors.New("invalid zero id") } @@ -271,13 +296,29 @@ func (b *blockWriter) append(id uint64) error { // element. b.data = binary.AppendUvarint(b.data, id-b.desc.max) } + // Extension validation + if (len(ext) == 0) != !b.hasExt { + if len(ext) == 0 { + return errors.New("missing extension") + } + return errors.New("unexpected extension") + } + // Append the extension if it is not nil. The extension is prefixed with a + // length indicator, and the block reader MUST understand this scheme and + // decode the extension accordingly. + if len(ext) > 0 { + b.setBitmap(ext) + enc := encodeIDs(ext) + b.data = binary.AppendUvarint(b.data, uint64(len(enc))) + b.data = append(b.data, enc...) + } b.desc.entries++ b.desc.max = id return nil } // scanSection traverses the specified section and terminates if fn returns true. -func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { +func (b *blockWriter) scanSection(section int, fn func(uint64, int, []uint16) bool) error { var ( value uint64 start = int(b.restarts[section]) @@ -296,27 +337,47 @@ func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { } else { value += x } - if fn(value, pos) { - return + // Resolve the extension if exists + var ( + err error + ext []uint16 + extLen int + ) + if b.hasExt { + l, ln := binary.Uvarint(b.data[pos+n:]) + extLen = ln + int(l) + ext, err = decodeIDs(b.data[pos+n+ln : pos+n+extLen]) + } + if err != nil { + return err + } + if fn(value, pos, ext) { + return nil } + // Shift to next position pos += n + pos += extLen } + return nil } // sectionLast returns the last element in the specified section. -func (b *blockWriter) sectionLast(section int) uint64 { +func (b *blockWriter) sectionLast(section int) (uint64, error) { var n uint64 - b.scanSection(section, func(v uint64, _ int) bool { + if err := b.scanSection(section, func(v uint64, _ int, _ []uint16) bool { n = v return false - }) - return n + }); err != nil { + return 0, err + } + return n, nil } // sectionSearch looks up the specified value in the given section, // the position and the preceding value will be returned if found. -func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int) { - b.scanSection(section, func(v uint64, p int) bool { +// It assumes that the preceding element exists in the section. +func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int, err error) { + if err := b.scanSection(section, func(v uint64, p int, _ []uint16) bool { if n == v { pos = p found = true @@ -324,8 +385,24 @@ func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uin } prev = v return false // continue iteration - }) - return found, prev, pos + }); err != nil { + return false, 0, 0, err + } + return found, prev, pos, nil +} + +// rebuildBitmap scans the entire block and rebuilds the bitmap. +func (b *blockWriter) rebuildBitmap() error { + clear(b.desc.extBitmap) + for i := 0; i < len(b.restarts); i++ { + if err := b.scanSection(i, func(v uint64, p int, ext []uint16) bool { + b.setBitmap(ext) + return false // continue iteration + }); err != nil { + return err + } + } + return nil } // pop removes the last element from the block. The assumption is held that block @@ -339,9 +416,9 @@ func (b *blockWriter) pop(id uint64) error { } // If there is only one entry left, the entire block should be reset if b.desc.entries == 1 { - //b.desc.min = 0 b.desc.max = 0 b.desc.entries = 0 + clear(b.desc.extBitmap) b.restarts = nil b.data = b.data[:0] return nil @@ -351,28 +428,45 @@ func (b *blockWriter) pop(id uint64) error { if b.desc.entries%indexBlockRestartLen == 1 { b.data = b.data[:b.restarts[len(b.restarts)-1]] b.restarts = b.restarts[:len(b.restarts)-1] - b.desc.max = b.sectionLast(len(b.restarts) - 1) + last, err := b.sectionLast(len(b.restarts) - 1) + if err != nil { + return err + } + b.desc.max = last b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } // Look up the element preceding the one to be popped, in order to update // the maximum element in the block. - found, prev, pos := b.sectionSearch(len(b.restarts)-1, id) + found, prev, pos, err := b.sectionSearch(len(b.restarts)-1, id) + if err != nil { + return err + } if !found { return fmt.Errorf("pop element is not found, last: %d, this: %d", b.desc.max, id) } b.desc.max = prev b.data = b.data[:pos] b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } func (b *blockWriter) empty() bool { return b.desc.empty() } -func (b *blockWriter) full() bool { - return b.desc.full() +func (b *blockWriter) estimateFull(ext []uint16) bool { + size := 8 + 2*len(ext) + return len(b.data)+size > indexBlockMaxSize +} + +// last returns the last element in the block. It should only be called when +// writer is not empty, otherwise the returned data is meaningless. +func (b *blockWriter) last() uint64 { + if b.empty() { + return 0 + } + return b.desc.max } // finish finalizes the index block encoding by appending the encoded restart points diff --git a/triedb/pathdb/history_index_block_test.go b/triedb/pathdb/history_index_block_test.go index c251cea2ec..923ae29348 100644 --- a/triedb/pathdb/history_index_block_test.go +++ b/triedb/pathdb/history_index_block_test.go @@ -17,6 +17,7 @@ package pathdb import ( + "bytes" "math" "math/rand" "slices" @@ -24,16 +25,36 @@ import ( "testing" ) +func randomExt(bitmapSize int, n int) []uint16 { + if bitmapSize == 0 { + return nil + } + var ( + limit = bitmapSize * 8 + extList []uint16 + ) + for i := 0; i < n; i++ { + extList = append(extList, uint16(rand.Intn(limit+1))) + } + return extList +} + func TestBlockReaderBasic(t *testing.T) { + testBlockReaderBasic(t, 0) + testBlockReaderBasic(t, 2) + testBlockReaderBasic(t, 34) +} + +func testBlockReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -60,18 +81,24 @@ func TestBlockReaderBasic(t *testing.T) { } func TestBlockReaderLarge(t *testing.T) { + testBlockReaderLarge(t, 0) + testBlockReaderLarge(t, 2) + testBlockReaderLarge(t, 34) +} + +func testBlockReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 for i := 0; i < 1000; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -95,34 +122,91 @@ func TestBlockReaderLarge(t *testing.T) { } func TestBlockWriterBasic(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriteBasic(t, 0) + testBlockWriteBasic(t, 2) + testBlockWriteBasic(t, 34) +} + +func testBlockWriteBasic(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) if !bw.empty() { t.Fatal("expected empty block") } - bw.append(2) - if err := bw.append(1); err == nil { + bw.append(2, randomExt(bitmapSize, 5)) + if err := bw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 3)) + bw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } - bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0)) + bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0, bitmapSize), maxElem, bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := bw.append(uint64(i + 100)); err != nil { + if err := bw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append value %d: %v", i, err) } } bw.finish() } +func TestBlockWriterWithLimit(t *testing.T) { + testBlockWriterWithLimit(t, 0) + testBlockWriterWithLimit(t, 2) + testBlockWriterWithLimit(t, 34) +} + +func testBlockWriterWithLimit(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) + + var bitmaps [][]byte + for i := 0; i < indexBlockRestartLen+2; i++ { + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) + bitmaps = append(bitmaps, bytes.Clone(bw.desc.extBitmap)) + } + for i := 0; i < indexBlockRestartLen+2; i++ { + limit := uint64(i + 1) + + desc := bw.desc.copy() + block, err := newBlockWriter(bytes.Clone(bw.finish()), desc, limit, bitmapSize != 0) + if err != nil { + t.Fatalf("Failed to construct the block writer, %v", err) + } + if block.desc.max != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, limit) + } + if !bytes.Equal(desc.extBitmap, bitmaps[i]) { + t.Fatalf("Test %d, unexpected bitmap, got: %v, want: %v", i, block.desc.extBitmap, bitmaps[i]) + } + + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < indexBlockRestartLen+4; elem++ { + if err := block.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if block.desc.max != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, maxElem) + } + } +} + func TestBlockWriterDelete(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriterDelete(t, 0) + testBlockWriterDelete(t, 2) + testBlockWriterDelete(t, 34) +} + +func testBlockWriterDelete(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) } // Pop unknown id, the request should be rejected if err := bw.pop(100); err == nil { @@ -144,12 +228,18 @@ func TestBlockWriterDelete(t *testing.T) { } func TestBlcokWriterDeleteWithData(t *testing.T) { + testBlcokWriterDeleteWithData(t, 0) + testBlcokWriterDeleteWithData(t, 2) + testBlcokWriterDeleteWithData(t, 34) +} + +func testBlcokWriterDeleteWithData(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } // Re-construct the block writer with data @@ -158,7 +248,10 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { max: 20, entries: 5, } - bw, err := newBlockWriter(bw.finish(), desc) + if bitmapSize > 0 { + desc.extBitmap = make([]byte, bitmapSize) + } + bw, err := newBlockWriter(bw.finish(), desc, elements[len(elements)-1], bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct block writer %v", err) } @@ -169,7 +262,7 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { newTail := elements[i-1] // Ensure the element can still be queried with no issue - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -201,26 +294,60 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { } func TestCorruptedIndexBlock(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) + + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), nil) + maxElem = uint64(i + 1) } buf := bw.finish() // Mutate the buffer manually buf[len(buf)-1]++ - _, err := newBlockWriter(buf, newIndexBlockDesc(0)) + _, err := newBlockWriter(buf, newIndexBlockDesc(0, 0), maxElem, false) if err == nil { t.Fatal("Corrupted index block data is not detected") } } // BenchmarkParseIndexBlock benchmarks the performance of parseIndexBlock. +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlock +// BenchmarkParseIndexBlock-8 35829495 34.16 ns/op func BenchmarkParseIndexBlock(b *testing.B) { // Generate a realistic index block blob - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) for i := 0; i < 4096; i++ { - bw.append(uint64(i * 2)) + bw.append(uint64(i*2), nil) + } + blob := bw.finish() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := parseIndexBlock(blob) + if err != nil { + b.Fatalf("parseIndexBlock failed: %v", err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlockWithExt +// BenchmarkParseIndexBlockWithExt-8 35773242 33.72 ns/op +func BenchmarkParseIndexBlockWithExt(b *testing.B) { + // Generate a realistic index block blob + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 34), 0, true) + for i := 0; i < 4096; i++ { + id, ext := uint64(i*2), randomExt(34, 3) + bw.append(id, ext) } blob := bw.finish() @@ -234,19 +361,58 @@ func BenchmarkParseIndexBlock(b *testing.B) { } // BenchmarkBlockWriterAppend benchmarks the performance of indexblock.writer +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppend +// BenchmarkBlockWriterAppend-8 293611083 4.113 ns/op 3 B/op 0 allocs/op func BenchmarkBlockWriterAppend(b *testing.B) { b.ReportAllocs() b.ResetTimer() - desc := newIndexBlockDesc(0) - writer, _ := newBlockWriter(nil, desc) + var blockID uint32 + desc := newIndexBlockDesc(blockID, 0) + writer, _ := newBlockWriter(nil, desc, 0, false) + + for i := 0; i < b.N; i++ { + if writer.estimateFull(nil) { + blockID += 1 + desc = newIndexBlockDesc(blockID, 0) + writer, _ = newBlockWriter(nil, desc, 0, false) + } + if err := writer.append(writer.desc.max+1, nil); err != nil { + b.Error(err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppendWithExt +// BenchmarkBlockWriterAppendWithExt-8 11123844 103.6 ns/op 42 B/op 2 allocs/op +func BenchmarkBlockWriterAppendWithExt(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + bitmapSize = 34 + blockID uint32 + ) + desc := newIndexBlockDesc(blockID, bitmapSize) + writer, _ := newBlockWriter(nil, desc, 0, true) for i := 0; i < b.N; i++ { - if writer.full() { - desc = newIndexBlockDesc(0) - writer, _ = newBlockWriter(nil, desc) + ext := randomExt(bitmapSize, 3) + if writer.estimateFull(ext) { + blockID += 1 + desc = newIndexBlockDesc(blockID, bitmapSize) + writer, _ = newBlockWriter(nil, desc, 0, true) } - if err := writer.append(writer.desc.max + 1); err != nil { + if err := writer.append(writer.desc.max+1, ext); err != nil { b.Error(err) } } diff --git a/triedb/pathdb/history_index_iterator.go b/triedb/pathdb/history_index_iterator.go new file mode 100644 index 0000000000..e4aca24f5d --- /dev/null +++ b/triedb/pathdb/history_index_iterator.go @@ -0,0 +1,656 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see = bitmapElementThresholdTwoLevels { + return false, fmt.Errorf("invalid extension filter %d for 2 bytes bitmap", id) + } + return isBitSet(bitmap, n), nil + case bitmapBytesThreeLevels: + // Bitmap for 3-level trie with at most 16+16*16 elements inside + if n >= bitmapElementThresholdThreeLevels { + return false, fmt.Errorf("invalid extension filter %d for 34 bytes bitmap", id) + } else if n >= bitmapElementThresholdTwoLevels { + return isBitSet(bitmap, n), nil + } else { + // Check the element itself first + if isBitSet(bitmap, n) { + return true, nil + } + // Check descendants: the presence of any descendant implicitly + // represents a mutation of its ancestor. + return bitmap[2+2*n] != 0 || bitmap[3+2*n] != 0, nil + } + default: + return false, fmt.Errorf("unsupported bitmap size %d", len(bitmap)) + } +} + +// blockIterator is the iterator to traverse the indices within a single block. +type blockIterator struct { + // immutable fields + data []byte // Reference to the data segment within the block reader + restarts []uint16 // Offsets pointing to the restart sections within the data + hasExt bool // Flag whether the extension is included in the data + + // Optional extension filter + filter *extFilter // Filters index entries based on the extension field. + + // mutable fields + id uint64 // ID of the element at the iterators current position + ext []byte // Extension field of the element at the iterators current position + dataPtr int // Current read position within the data slice + restartPtr int // Index of the restart section where the iterator is currently positioned + exhausted bool // Flag whether the iterator has been exhausted + err error // Accumulated error during the traversal +} + +func (br *blockReader) newIterator(filter *extFilter) *blockIterator { + it := &blockIterator{ + data: br.data, // hold the slice directly with no deep copy + restarts: br.restarts, // hold the slice directly with no deep copy + hasExt: br.hasExt, // flag whether the extension should be resolved + filter: filter, // optional extension filter + } + it.reset() + return it +} + +func (it *blockIterator) set(dataPtr int, restartPtr int, id uint64, ext []byte) { + it.id = id + it.ext = ext + + it.dataPtr = dataPtr + it.restartPtr = restartPtr + it.exhausted = dataPtr == len(it.data) +} + +func (it *blockIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *blockIterator) reset() { + it.id = 0 + it.ext = nil + + it.dataPtr = -1 + it.restartPtr = -1 + it.exhausted = false + it.err = nil + + // Mark the iterator as exhausted if the associated index block is empty + if len(it.data) == 0 || len(it.restarts) == 0 { + it.exhausted = true + } +} + +func (it *blockIterator) resolveExt(pos int) ([]byte, int, error) { + if !it.hasExt { + return nil, 0, nil + } + length, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, datalen: %d", pos, len(it.data)) + } + if len(it.data[pos+n:]) < int(length) { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, length: %d, datalen: %d", pos, length, len(it.data)) + } + return it.data[pos+n : pos+n+int(length)], n + int(length), nil +} + +// seekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *blockIterator) seekGT(id uint64) bool { + if it.err != nil { + return false + } + var err error + index := sort.Search(len(it.restarts), func(i int) bool { + item, n := binary.Uvarint(it.data[it.restarts[i]:]) + if n <= 0 { + err = fmt.Errorf("failed to decode item at restart %d", it.restarts[i]) + } + return item > id + }) + if err != nil { + it.setErr(err) + return false + } + if index == 0 { + pos := int(it.restarts[0]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[0])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, 0, item, ext) + return true + } + var ( + start int + limit int + restartIndex int // The restart section being searched below + ) + if index == len(it.restarts) { + // The element being searched falls within the last restart section, + // there is no guarantee such element can be found. + start = int(it.restarts[len(it.restarts)-1]) + limit = len(it.data) + restartIndex = len(it.restarts) - 1 + } else { + // The element being searched falls within the non-last restart section, + // such element can be found for sure. + start = int(it.restarts[index-1]) + limit = int(it.restarts[index]) + restartIndex = index - 1 + } + var ( + result uint64 + pos = start + ) + for pos < limit { + x, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", pos)) + return false + } + if pos == start { + result = x + } else { + result += x + } + pos += n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + pos += shift + + if result > id { + if pos == limit { + it.set(pos, restartIndex+1, result, ext) + } else { + it.set(pos, restartIndex, result, ext) + } + return true + } + } + // The element which is greater than specified id is not found. + if index == len(it.restarts) { + it.reset() + return false + } + // The element which is the first one greater than the specified id + // is exactly the one located at the restart point. + pos = int(it.restarts[index]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[index])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, index, item, ext) + return true +} + +// SeekGT implements HistoryIndexIterator, is the wrapper of the seekGT with +// optional extension filter logic applied. +func (it *blockIterator) SeekGT(id uint64) bool { + if !it.seekGT(id) { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +func (it *blockIterator) init() { + if it.dataPtr != -1 { + return + } + it.dataPtr = 0 + it.restartPtr = 0 +} + +// next moves the iterator to the next element. If the iterator has been exhausted, +// and boolean with false should be returned. +func (it *blockIterator) next() bool { + if it.exhausted || it.err != nil { + return false + } + it.init() + + // Decode the next element pointed by the iterator + v, n := binary.Uvarint(it.data[it.dataPtr:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.dataPtr)) + return false + } + var val uint64 + if it.dataPtr == int(it.restarts[it.restartPtr]) { + val = v + } else { + val = it.id + v + } + + // Decode the extension field + ext, shift, err := it.resolveExt(it.dataPtr + n) + if err != nil { + it.setErr(err) + return false + } + + // Move to the next restart section if the data pointer crosses the boundary + nextRestartPtr := it.restartPtr + if it.restartPtr < len(it.restarts)-1 && it.dataPtr+n+shift == int(it.restarts[it.restartPtr+1]) { + nextRestartPtr = it.restartPtr + 1 + } + it.set(it.dataPtr+n+shift, nextRestartPtr, val, ext) + + return true +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. It's a wrapper of next with optional extension filter logic applied. +func (it *blockIterator) Next() bool { + if !it.next() { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *blockIterator) ID() uint64 { + return it.id +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *blockIterator) Error() error { return it.err } + +// indexIterator is an iterator to traverse the history indices belonging to the +// specific state entry. +type indexIterator struct { + // immutable fields + descList []*indexBlockDesc + reader *indexReader + + // Optional extension filter + filter *extFilter + + // mutable fields + blockIt *blockIterator + blockPtr int + exhausted bool + err error +} + +// newBlockIter initializes the block iterator with the specified block ID. +func (r *indexReader) newBlockIter(id uint32, filter *extFilter) (*blockIterator, error) { + br, ok := r.readers[id] + if !ok { + var err error + br, err = newBlockReader(readStateIndexBlock(r.state, r.db, id), r.bitmapSize != 0) + if err != nil { + return nil, err + } + r.readers[id] = br + } + return br.newIterator(filter), nil +} + +// newIterator initializes the index iterator with the specified extension filter. +func (r *indexReader) newIterator(filter *extFilter) *indexIterator { + it := &indexIterator{ + descList: r.descList, + reader: r, + filter: filter, + } + it.reset() + return it +} + +func (it *indexIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *indexIterator) reset() { + it.blockIt = nil + it.blockPtr = -1 + it.exhausted = false + it.err = nil + + if len(it.descList) == 0 { + it.exhausted = true + } +} + +func (it *indexIterator) open(blockPtr int) error { + blockIt, err := it.reader.newBlockIter(it.descList[blockPtr].id, it.filter) + if err != nil { + return err + } + it.blockIt = blockIt + it.blockPtr = blockPtr + return nil +} + +func (it *indexIterator) applyFilter(index int) (int, error) { + if it.filter == nil { + return index, nil + } + for index < len(it.descList) { + found, err := it.filter.contains(it.descList[index].extBitmap) + if err != nil { + return 0, err + } + if found { + break + } + index++ + } + return index, nil +} + +// SeekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *indexIterator) SeekGT(id uint64) bool { + if it.err != nil { + return false + } + index := sort.Search(len(it.descList), func(i int) bool { + return id < it.descList[i].max + }) + index, err := it.applyFilter(index) + if err != nil { + it.setErr(err) + return false + } + if index == len(it.descList) { + return false + } + it.exhausted = false + + if it.blockIt == nil || it.blockPtr != index { + if err := it.open(index); err != nil { + it.setErr(err) + return false + } + } + // Terminate if the element which is greater than the id can be found in the + // last block; otherwise move to the next block. It may happen that all the + // target elements in this block are all less than id. + if it.blockIt.SeekGT(id) { + return true + } + return it.Next() +} + +func (it *indexIterator) init() error { + if it.blockIt != nil { + return nil + } + return it.open(0) +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. If the iterator has been exhausted, and boolean with false should +// be returned. +func (it *indexIterator) Next() bool { + if it.exhausted || it.err != nil { + return false + } + if err := it.init(); err != nil { + it.setErr(err) + return false + } + if it.blockIt.Next() { + return true + } + it.blockPtr++ + + index, err := it.applyFilter(it.blockPtr) + if err != nil { + it.setErr(err) + return false + } + it.blockPtr = index + + if it.blockPtr == len(it.descList) { + it.exhausted = true + return false + } + if err := it.open(it.blockPtr); err != nil { + it.setErr(err) + return false + } + return it.blockIt.Next() +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *indexIterator) Error() error { + if it.err != nil { + return it.err + } + if it.blockIt != nil { + return it.blockIt.Error() + } + return nil +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *indexIterator) ID() uint64 { + return it.blockIt.ID() +} + +// seqIter provides a simple iterator over a contiguous sequence of +// unsigned integers, ending at end (end is included). +type seqIter struct { + cur uint64 // current position + end uint64 // iteration stops at end-1 + done bool // true when iteration is exhausted +} + +func newSeqIter(last uint64) *seqIter { + return &seqIter{end: last + 1} +} + +// SeekGT positions the iterator at the smallest element > id. +// Returns false if no such element exists. +func (it *seqIter) SeekGT(id uint64) bool { + if id+1 >= it.end { + it.done = true + return false + } + it.cur = id + 1 + it.done = false + return true +} + +// Next advances the iterator. Returns false if exhausted. +func (it *seqIter) Next() bool { + if it.done { + return false + } + if it.cur+1 < it.end { + it.cur++ + return true + } + // this was the last element + it.done = true + return false +} + +// ID returns the id of the element where the iterator is positioned at. +func (it *seqIter) ID() uint64 { return it.cur } + +// Error returns any accumulated error. Exhausting all the elements is not +// considered to be an error. +func (it *seqIter) Error() error { return nil } diff --git a/triedb/pathdb/history_index_iterator_test.go b/triedb/pathdb/history_index_iterator_test.go new file mode 100644 index 0000000000..a11fd17666 --- /dev/null +++ b/triedb/pathdb/history_index_iterator_test.go @@ -0,0 +1,361 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + + var input uint64 + if rand.Intn(2) == 0 { + input = elements[rand.Intn(len(elements))] + } else { + input = uint64(rand.Uint32()) + } + + index := sort.Search(len(elements), func(i int) bool { + return elements[i] > input + }) + for index < len(elements) { + if checkExt(filter, ext[index]) { + break + } + index++ + } + + var ( + exp bool + expVal uint64 + remains []uint64 + ) + if index == len(elements) { + exp = false + } else { + exp = true + expVal = elements[index] + + index++ + for index < len(elements) { + if checkExt(filter, ext[index]) { + remains = append(remains, elements[index]) + } + index++ + } + } + + it := newIter(filter) + if err := checkSeekGT(it, input, exp, expVal); err != nil { + t.Fatal(err) + } + if exp { + if err := checkNext(it, remains); err != nil { + t.Fatal(err) + } + } + } +} + +func verifyTraversal(t *testing.T, elements []uint64, ext [][]uint16, newIter func(filter *extFilter) HistoryIndexIterator) { + set := make(map[extFilter]bool) + for _, extList := range ext { + for _, f := range extList { + set[extFilter(f)] = true + } + } + filters := slices.Collect(maps.Keys(set)) + + for i := 0; i < 16; i++ { + var filter *extFilter + if len(filters) > 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + it := newIter(filter) + + var ( + pos int + exp []uint64 + ) + for pos < len(elements) { + if checkExt(filter, ext[pos]) { + exp = append(exp, elements[pos]) + } + pos++ + } + if err := checkNext(it, exp); err != nil { + t.Fatal(err) + } + } +} + +func TestBlockIteratorSeekGT(t *testing.T) { + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorSeekGT(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestBlockIteratorTraversal(t *testing.T) { + /* 0-size index block is not allowed + + data, elements := makeTestIndexBlock(0) + testBlockIterator(t, data, elements) + */ + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorTraversal(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestSeqIterBasicIteration(t *testing.T) { + it := newSeqIter(5) // iterates over [1..5] + it.SeekGT(0) + + var ( + got []uint64 + expected = []uint64{1, 2, 3, 4, 5} + ) + got = append(got, it.ID()) + for it.Next() { + got = append(got, it.ID()) + } + if len(got) != len(expected) { + t.Fatalf("iteration length mismatch: got %v, expected %v", got, expected) + } + for i := range expected { + if got[i] != expected[i] { + t.Fatalf("element mismatch at %d: got %d, expected %d", i, got[i], expected[i]) + } + } +} + +func TestSeqIterSeekGT(t *testing.T) { + it := newSeqIter(5) + + tests := []struct { + input uint64 + ok bool + expected uint64 + }{ + {0, true, 1}, + {1, true, 2}, + {4, true, 5}, + {5, false, 0}, // 6 is out of range + } + for _, tt := range tests { + ok := it.SeekGT(tt.input) + if ok != tt.ok { + t.Fatalf("SeekGT(%d) ok mismatch: got %v, expected %v", tt.input, ok, tt.ok) + } + if ok && it.ID() != tt.expected { + t.Fatalf("SeekGT(%d) positioned at %d, expected %d", tt.input, it.ID(), tt.expected) + } + } +} diff --git a/triedb/pathdb/history_index_test.go b/triedb/pathdb/history_index_test.go index be9b7c4049..2644db46b5 100644 --- a/triedb/pathdb/history_index_test.go +++ b/triedb/pathdb/history_index_test.go @@ -29,19 +29,25 @@ import ( ) func TestIndexReaderBasic(t *testing.T) { + testIndexReaderBasic(t, 0) + testIndexReaderBasic(t, 2) + testIndexReaderBasic(t, 34) +} + +func testIndexReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -68,22 +74,28 @@ func TestIndexReaderBasic(t *testing.T) { } func TestIndexReaderLarge(t *testing.T) { + testIndexReaderLarge(t, 0) + testIndexReaderLarge(t, 2) + testIndexReaderLarge(t, 34) +} + +func testIndexReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 - for i := 0; i < 10*indexBlockEntriesCap; i++ { + for i := 0; i < 10*4096; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -107,7 +119,7 @@ func TestIndexReaderLarge(t *testing.T) { } func TestEmptyIndexReader(t *testing.T) { - br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa}), 0) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -121,57 +133,150 @@ func TestEmptyIndexReader(t *testing.T) { } func TestIndexWriterBasic(t *testing.T) { + testIndexWriterBasic(t, 0) + testIndexWriterBasic(t, 2) + testIndexWriterBasic(t, 34) +} + +func testIndexWriterBasic(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - iw.append(2) - if err := iw.append(1); err == nil { + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + iw.append(2, randomExt(bitmapSize, 5)) + if err := iw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - iw.append(uint64(i + 3)) + iw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } batch := db.NewBatch() iw.finish(batch) batch.Write() - iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := iw.append(uint64(i + 100)); err != nil { + if err := iw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append item, %v", err) } } iw.finish(db.NewBatch()) } -func TestIndexWriterDelete(t *testing.T) { +func TestIndexWriterWithLimit(t *testing.T) { + testIndexWriterWithLimit(t, 0) + testIndexWriterWithLimit(t, 2) + testIndexWriterWithLimit(t, 34) +} + +func testIndexWriterWithLimit(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - for i := 0; i < indexBlockEntriesCap*4; i++ { - iw.append(uint64(i + 1)) + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if iw.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < 500; elem++ { + if err := iw.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if iw.lastID != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, maxElem) + } + } +} + +func TestIndexDeleterBasic(t *testing.T) { + testIndexDeleterBasic(t, 0) + testIndexDeleterBasic(t, 2) + testIndexDeleterBasic(t, 34) +} + +func testIndexDeleterBasic(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + var maxElem uint64 + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + maxElem = uint64(i + 1) } batch := db.NewBatch() iw.finish(batch) batch.Write() // Delete unknown id, the request should be rejected - id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa})) - if err := id.pop(indexBlockEntriesCap * 5); err == nil { + id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) + if err := id.pop(500); err == nil { t.Fatal("Expect error to occur for unknown id") } - for i := indexBlockEntriesCap * 4; i >= 1; i-- { + for i := 200; i >= 1; i-- { if err := id.pop(uint64(i)); err != nil { t.Fatalf("Unexpected error for element popping, %v", err) } if id.lastID != uint64(i-1) { t.Fatalf("Unexpected lastID, want: %d, got: %d", uint64(i-1), iw.lastID) } - if rand.Intn(10) == 0 { - batch := db.NewBatch() - id.finish(batch) - batch.Write() + } +} + +func TestIndexDeleterWithLimit(t *testing.T) { + testIndexDeleterWithLimit(t, 0) + testIndexDeleterWithLimit(t, 2) + testIndexDeleterWithLimit(t, 34) +} + +func testIndexDeleterWithLimit(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + id, err := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if id.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Keep removing elements + for elem := id.lastID; elem > 0; elem-- { + if err := id.pop(elem); err != nil { + t.Fatalf("Failed to pop value %d: %v", elem, err) + } } } } @@ -212,7 +317,7 @@ func TestBatchIndexerWrite(t *testing.T) { } } for addrHash, indexes := range accounts { - ir, _ := newIndexReader(db, newAccountIdent(addrHash)) + ir, _ := newIndexReader(db, newAccountIdent(addrHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { @@ -232,7 +337,7 @@ func TestBatchIndexerWrite(t *testing.T) { } for addrHash, slots := range storages { for slotHash, indexes := range slots { - ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash)) + ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index 893ccd6523..c987b380ed 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -34,17 +34,13 @@ import ( const ( // The batch size for reading state histories - historyReadBatch = 1000 + historyReadBatch = 1000 + historyIndexBatch = 8 * 1024 * 1024 // The number of state history indexes for constructing or deleting as batch stateHistoryIndexV0 = uint8(0) // initial version of state index structure stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version - - // estimations for calculating the batch size for atomic database commit - estimatedStateHistoryIndexSize = 3 // The average size of each state history index entry is approximately 2–3 bytes - estimatedTrienodeHistoryIndexSize = 3 // The average size of each trienode history index entry is approximately 2-3 bytes - estimatedIndexBatchSizeFactor = 32 // The factor counts for the write amplification for each entry ) // indexVersion returns the latest index version for the given history type. @@ -125,18 +121,20 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) { // batchIndexer is responsible for performing batch indexing or unindexing // of historical data (e.g., state or trie node changes) atomically. type batchIndexer struct { - index map[stateIdent][]uint64 // List of history IDs for tracked state entry - pending int // Number of entries processed in the current batch. - delete bool // Operation mode: true for unindex, false for index. - lastID uint64 // ID of the most recently processed history. - typ historyType // Type of history being processed (e.g., state or trienode). - db ethdb.KeyValueStore // Key-value database used to store or delete index data. + index map[stateIdent][]uint64 // List of history IDs for tracked state entry + ext map[stateIdent][][]uint16 // List of extension for each state element + pending int // Number of entries processed in the current batch. + delete bool // Operation mode: true for unindex, false for index. + lastID uint64 // ID of the most recently processed history. + typ historyType // Type of history being processed (e.g., state or trienode). + db ethdb.KeyValueStore // Key-value database used to store or delete index data. } // newBatchIndexer constructs the batch indexer with the supplied mode. func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batchIndexer { return &batchIndexer{ index: make(map[stateIdent][]uint64), + ext: make(map[stateIdent][][]uint16), delete: delete, typ: typ, db: db, @@ -146,8 +144,10 @@ func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batc // process traverses the state entries within the provided history and tracks the mutation // records for them. func (b *batchIndexer) process(h history, id uint64) error { - for ident := range h.forEach() { - b.index[ident] = append(b.index[ident], id) + for elem := range h.forEach() { + key := elem.key() + b.index[key] = append(b.index[key], id) + b.ext[key] = append(b.ext[key], elem.ext()) b.pending++ } b.lastID = id @@ -155,22 +155,6 @@ func (b *batchIndexer) process(h history, id uint64) error { return b.finish(false) } -// makeBatch constructs a database batch based on the number of pending entries. -// The batch size is roughly estimated to minimize repeated resizing rounds, -// as accurately predicting the exact size is technically challenging. -func (b *batchIndexer) makeBatch() ethdb.Batch { - var size int - switch b.typ { - case typeStateHistory: - size = estimatedStateHistoryIndexSize - case typeTrienodeHistory: - size = estimatedTrienodeHistoryIndexSize - default: - panic(fmt.Sprintf("unknown history type %d", b.typ)) - } - return b.db.NewBatchWithSize(size * estimatedIndexBatchSizeFactor * b.pending) -} - // finish writes the accumulated state indexes into the disk if either the // memory limitation is reached or it's requested forcibly. func (b *batchIndexer) finish(force bool) error { @@ -181,30 +165,52 @@ func (b *batchIndexer) finish(force bool) error { return nil } var ( - batch = b.makeBatch() - batchMu sync.RWMutex - start = time.Now() - eg errgroup.Group + start = time.Now() + eg errgroup.Group + + batch = b.db.NewBatchWithSize(ethdb.IdealBatchSize) + batchSize int + batchMu sync.RWMutex + + writeBatch = func(fn func(batch ethdb.Batch)) error { + batchMu.Lock() + defer batchMu.Unlock() + + fn(batch) + if batch.ValueSize() >= ethdb.IdealBatchSize { + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + } + return nil + } ) eg.SetLimit(runtime.NumCPU()) + var indexed uint64 + if metadata := loadIndexMetadata(b.db, b.typ); metadata != nil { + indexed = metadata.Last + } for ident, list := range b.index { + ext := b.ext[ident] eg.Go(func() error { if !b.delete { - iw, err := newIndexWriter(b.db, ident) + iw, err := newIndexWriter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } - for _, n := range list { - if err := iw.append(n); err != nil { + for i, n := range list { + if err := iw.append(n, ext[i]); err != nil { return err } } - batchMu.Lock() - iw.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + iw.finish(batch) + }) } else { - id, err := newIndexDeleter(b.db, ident) + id, err := newIndexDeleter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } @@ -213,11 +219,10 @@ func (b *batchIndexer) finish(force bool) error { return err } } - batchMu.Lock() - id.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + id.finish(batch) + }) } - return nil }) } if err := eg.Wait(); err != nil { @@ -233,12 +238,16 @@ func (b *batchIndexer) finish(force bool) error { storeIndexMetadata(batch, b.typ, b.lastID-1) } } + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { return err } - log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "size", common.StorageSize(batchSize), "elapsed", common.PrettyDuration(time.Since(start))) + b.pending = 0 - b.index = make(map[stateIdent][]uint64) + clear(b.index) + clear(b.ext) return nil } diff --git a/triedb/pathdb/history_inspect.go b/triedb/pathdb/history_inspect.go index 9b4eea27b4..74b8bb8df2 100644 --- a/triedb/pathdb/history_inspect.go +++ b/triedb/pathdb/history_inspect.go @@ -50,12 +50,12 @@ func sanitizeRange(start, end uint64, freezer ethdb.AncientReader) (uint64, uint if err != nil { return 0, 0, err } - last := head - 1 + last := head if end != 0 && end < last { last = end } // Make sure the range is valid - if first >= last { + if first > last { return 0, 0, fmt.Errorf("range is invalid, first: %d, last: %d", first, last) } return first, last, nil @@ -143,7 +143,7 @@ func historyRange(freezer ethdb.AncientReader) (uint64, uint64, error) { if err != nil { return 0, 0, err } - last := head - 1 + last := head fh, err := readStateHistory(freezer, first) if err != nil { diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index 1bf4cf648d..4ae1fb36cb 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -22,11 +22,17 @@ import ( "errors" "fmt" "math" + "slices" "sort" + "sync" + "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/sync/errgroup" ) // indexReaderWithLimitTag is a wrapper around indexReader that includes an @@ -40,8 +46,8 @@ type indexReaderWithLimitTag struct { } // newIndexReaderWithLimitTag constructs a index reader with indexing position. -func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*indexReaderWithLimitTag, error) { - r, err := newIndexReader(db, state) +func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexReaderWithLimitTag, error) { + r, err := newIndexReader(db, state, bitmapSize) if err != nil { return nil, err } @@ -99,16 +105,17 @@ func (r *indexReaderWithLimitTag) readGreaterThan(id uint64, lastID uint64) (uin return r.reader.readGreaterThan(id) } -// historyReader is the structure to access historic state data. -type historyReader struct { +// stateHistoryReader is the structure to access historic state data. +type stateHistoryReader struct { disk ethdb.KeyValueReader freezer ethdb.AncientReader readers map[string]*indexReaderWithLimitTag } -// newHistoryReader constructs the history reader with the supplied db. -func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *historyReader { - return &historyReader{ +// newStateHistoryReader constructs the history reader with the supplied db +// for accessing historical states. +func newStateHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *stateHistoryReader { + return &stateHistoryReader{ disk: disk, freezer: freezer, readers: make(map[string]*indexReaderWithLimitTag), @@ -117,7 +124,7 @@ func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *h // readAccountMetadata resolves the account metadata within the specified // state history. -func (r *historyReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { blob := rawdb.ReadStateAccountIndex(r.freezer, historyID) if len(blob) == 0 { return nil, fmt.Errorf("account index is truncated, historyID: %d", historyID) @@ -143,7 +150,7 @@ func (r *historyReader) readAccountMetadata(address common.Address, historyID ui // readStorageMetadata resolves the storage slot metadata within the specified // state history. -func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { +func (r *stateHistoryReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { data, err := rawdb.ReadStateStorageIndex(r.freezer, historyID, slotIndexSize*slotOffset, slotIndexSize*slotNumber) if err != nil { msg := fmt.Sprintf("id: %d, slot-offset: %d, slot-length: %d", historyID, slotOffset, slotNumber) @@ -178,7 +185,7 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash } // readAccount retrieves the account data from the specified state history. -func (r *historyReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -194,7 +201,7 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64) ([ } // readStorage retrieves the storage slot data from the specified state history. -func (r *historyReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -224,35 +231,16 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha // stateID: represents the ID of the state of the specified version; // lastID: represents the ID of the latest/newest state history; // latestValue: represents the state value at the current disk layer with ID == lastID; -func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { - tail, err := r.freezer.Tail() +func (r *stateHistoryReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + lastIndexed, err := checkStateAvail(state.stateIdent, typeStateHistory, r.freezer, stateID, lastID, r.disk) if err != nil { return nil, err - } // firstID = tail+1 - - // stateID+1 == firstID is allowed, as all the subsequent state histories - // are present with no gap inside. - if stateID < tail { - return nil, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) } - - // To serve the request, all state histories from stateID+1 to lastID - // must be indexed. It's not supposed to happen unless system is very - // wrong. - metadata := loadIndexMetadata(r.disk, toHistoryType(state.typ)) - if metadata == nil || metadata.Last < lastID { - indexed := "null" - if metadata != nil { - indexed = fmt.Sprintf("%d", metadata.Last) - } - return nil, fmt.Errorf("state history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) - } - // Construct the index reader to locate the corresponding history for // state retrieval ir, ok := r.readers[state.String()] if !ok { - ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, metadata.Last) + ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, lastIndexed, 0) if err != nil { return nil, err } @@ -277,3 +265,230 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 } return r.readStorage(state.address, state.storageKey, state.storageHash, historyID) } + +// trienodeReader is the structure to access historical trienode data. +type trienodeReader struct { + disk ethdb.KeyValueReader + freezer ethdb.AncientReader + readConcurrency int // The concurrency used to load trie node data from history +} + +// newTrienodeReader constructs the history reader with the supplied db +// for accessing historical trie nodes. +func newTrienodeReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader, readConcurrency int) *trienodeReader { + return &trienodeReader{ + disk: disk, + freezer: freezer, + readConcurrency: readConcurrency, + } +} + +// readTrienode retrieves the trienode data from the specified trienode history. +func (r *trienodeReader) readTrienode(addrHash common.Hash, path string, historyID uint64) ([]byte, bool, error) { + tr := newTrienodeHistoryReader(historyID, r.freezer) + return tr.read(addrHash, path) +} + +// assembleNode takes a complete node value as the base and applies a list of +// mutation records to assemble the final node value accordingly. +func assembleNode(blob []byte, elements [][][]byte, indices [][]int) ([]byte, error) { + if len(elements) == 0 && len(indices) == 0 { + return blob, nil + } + children, err := rlp.SplitListValues(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(elements); i++ { + for j, pos := range indices[i] { + children[pos] = elements[i][j] + } + } + return rlp.MergeListValues(children) +} + +type resultQueue struct { + data [][]byte + lock sync.Mutex +} + +func newResultQueue(size int) *resultQueue { + return &resultQueue{ + data: make([][]byte, size, size*2), + } +} + +func (q *resultQueue) set(data []byte, pos int) { + q.lock.Lock() + defer q.lock.Unlock() + + if pos >= len(q.data) { + newSize := pos + 1 + if cap(q.data) < newSize { + newData := make([][]byte, newSize, newSize*2) + copy(newData, q.data) + q.data = newData + } + q.data = q.data[:newSize] + } + q.data[pos] = data +} + +func (r *trienodeReader) readOptimized(state stateIdent, it HistoryIndexIterator, latestValue []byte) ([]byte, error) { + var ( + elements [][][]byte + indices [][]int + blob = latestValue + + eg errgroup.Group + seq int + term atomic.Bool + queue = newResultQueue(r.readConcurrency * 2) + ) + eg.SetLimit(r.readConcurrency) + + for { + id, pos := it.ID(), seq + seq += 1 + + eg.Go(func() error { + data, found, err := r.readTrienode(state.addressHash, state.path, id) + if err != nil { + term.Store(true) + return err + } + // In optimistic readahead mode, it is theoretically possible to encounter a + // NotFound error, where the trie node does not actually exist and the iterator + // reports a false-positive mutation record. Terminate the iterator if so, as + // all the necessary data (checkpoints and all diffs) required has already been + // fetching. + if !found { + term.Store(true) + log.Debug("Failed to read the trienode") + return nil + } + full, _, err := decodeNodeFull(data) + if err != nil { + term.Store(true) + return err + } + if full { + term.Store(true) + } + queue.set(data, pos) + return nil + }) + if term.Load() || !it.Next() { + break + } + } + if err := eg.Wait(); err != nil { + return nil, err + } + if err := it.Error(); err != nil { + return nil, err + } + for i := 0; i < seq; i++ { + isComplete, fullBlob, err := decodeNodeFull(queue.data[i]) + if err != nil { + return nil, err + } + // Terminate the loop is the node with full value has been found + if isComplete { + blob = fullBlob + break + } + // Decode the partial encoded node and keep iterating the node history + // until the node with full value being reached. + element, index, err := decodeNodeCompressed(queue.data[i]) + if err != nil { + return nil, err + } + elements, indices = append(elements, element), append(indices, index) + } + slices.Reverse(elements) + slices.Reverse(indices) + return assembleNode(blob, elements, indices) +} + +// read retrieves the trie node data associated with the stateID. +// stateID: represents the ID of the state of the specified version; +// lastID: represents the ID of the latest/newest trie node history; +// latestValue: represents the trie node value at the current disk layer with ID == lastID; +func (r *trienodeReader) read(state stateIdent, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + _, err := checkStateAvail(state, typeTrienodeHistory, r.freezer, stateID, lastID, r.disk) + if err != nil { + return nil, err + } + // Construct the index iterator to traverse the trienode history + var ( + scheme *indexScheme + it HistoryIndexIterator + ) + if state.addressHash == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + if state.addressHash == (common.Hash{}) && state.path == "" { + it = newSeqIter(lastID) + } else { + chunkID, nodeID := scheme.splitPathLast(state.path) + + queryIdent := state + queryIdent.path = chunkID + ir, err := newIndexReader(r.disk, queryIdent, scheme.getBitmapSize(len(chunkID))) + if err != nil { + return nil, err + } + filter := extFilter(nodeID) + it = ir.newIterator(&filter) + } + // Move the iterator to the first element whose id is greater than + // the given number. + found := it.SeekGT(stateID) + if err := it.Error(); err != nil { + return nil, err + } + // The state was not found in the trie node histories, as it has not been + // modified since stateID. Use the data from the associated disk layer + // instead (full value node as always) + if !found { + return latestValue, nil + } + return r.readOptimized(state, it, latestValue) +} + +// checkStateAvail determines whether the requested historical state is available +// for accessing. What's more, it also returns the ID of the latest indexed history +// entry for subsequent usage. +// +// TODO(rjl493456442) it's really expensive to perform the check for every state +// retrieval, please rework this later. +func checkStateAvail(state stateIdent, exptyp historyType, freezer ethdb.AncientReader, stateID uint64, lastID uint64, db ethdb.KeyValueReader) (uint64, error) { + if toHistoryType(state.typ) != exptyp { + return 0, fmt.Errorf("unsupported history type: %d, want: %v", toHistoryType(state.typ), exptyp) + } + // firstID = tail+1 + tail, err := freezer.Tail() + if err != nil { + return 0, err + } + // stateID+1 == firstID is allowed, as all the subsequent history entries + // are present with no gap inside. + if stateID < tail { + return 0, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) + } + // To serve the request, all history entries from stateID+1 to lastID + // must be indexed. It's not supposed to happen unless system is very + // wrong. + metadata := loadIndexMetadata(db, exptyp) + if metadata == nil || metadata.Last < lastID { + indexed := "null" + if metadata != nil { + indexed = fmt.Sprintf("%d", metadata.Last) + } + return 0, fmt.Errorf("history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) + } + return metadata.Last, nil +} diff --git a/triedb/pathdb/history_reader_test.go b/triedb/pathdb/history_reader_test.go index dc2894e338..5a1ed28bd2 100644 --- a/triedb/pathdb/history_reader_test.go +++ b/triedb/pathdb/history_reader_test.go @@ -50,7 +50,7 @@ func stateAvail(id uint64, env *tester) bool { return id+1 >= firstID } -func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *historyReader) error { +func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *stateHistoryReader) error { if !stateAvail(id, env) { return nil } @@ -153,7 +153,7 @@ func testHistoryReader(t *testing.T, historyLimit uint64) { var ( roots = env.roots dl = env.db.tree.bottom() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dl.rootHash() { diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index bc21915dba..23428b1a54 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -283,11 +283,11 @@ func (h *stateHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *stateHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *stateHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, addr := range h.accountList { addrHash := crypto.Keccak256Hash(addr.Bytes()) - if !yield(newAccountIdent(addrHash)) { + if !yield(accountIndexElem{addrHash}) { return } for _, slotKey := range h.storageList[addr] { @@ -298,7 +298,7 @@ func (h *stateHistory) forEach() iter.Seq[stateIdent] { if h.meta.version != stateHistoryV0 { slotHash = crypto.Keccak256Hash(slotKey.Bytes()) } - if !yield(newStorageIdent(addrHash, slotHash)) { + if !yield(storageIndexElem{addrHash, slotHash}) { return } } diff --git a/triedb/pathdb/history_trienode.go b/triedb/pathdb/history_trienode.go index 3f45b41117..11d6112806 100644 --- a/triedb/pathdb/history_trienode.go +++ b/triedb/pathdb/history_trienode.go @@ -19,9 +19,11 @@ package pathdb import ( "bytes" "encoding/binary" + "errors" "fmt" "iter" "maps" + "math" "slices" "sort" "time" @@ -44,7 +46,10 @@ import ( // - block number (8 bytes) // // - a lexicographically sorted list of trie IDs -// - the corresponding offsets into the key and value sections for each trie data chunk +// - the corresponding offsets into the key and value sections for each trie +// data chunk. The offsets refer to the end position of each chunk, with +// the assumption that the key and value sections for the first data chunk +// start at offset 0. // // Although some fields (e.g., parent state root, block number) are duplicated // between the state history and the trienode history, these two histories @@ -53,19 +58,16 @@ import ( // // # Key section // The key section stores trie node keys (paths) in a compressed format. -// It also contains relative offsets into the value section for resolving -// the corresponding trie node data. Note that these offsets are relative -// to the data chunk for the trie; the chunk offset must be added to obtain -// the absolute position. +// It also contains relative offsets into the value section for locating +// the corresponding trie node data. These offsets are relative to the +// beginning of the trie data chunk, the chunk's base offset must be added +// to obtain the absolute position in the value section. // // # Value section // The value section is a concatenated byte stream of all trie node data. // Each trie node can be retrieved using the offset and length specified // by its index entry. // -// The header and key sections are sufficient for locating a trie node, -// while a partial read of the value section is enough to retrieve its data. - // Header section: // // +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------| @@ -87,9 +89,9 @@ import ( // // +---- key len ----+ // / \ -// +-------+---------+-----------+---------+-----------------------+-----------------+ -// | shared (varint) | not shared (varint) | value length (varlen) | key (varlen) | -// +-----------------+---------------------+-----------------------+-----------------+ +// +-------+---------+-----------+---------+-----------------------+-----------------------+ +// | shared (varint) | not shared (varint) | value length (varlen) | unshared key (varlen) | +// +-----------------+---------------------+-----------------------+-----------------------+ // // trailer: // @@ -99,9 +101,9 @@ import ( // | restart_1 key offset | restart_1 value offset | ... | restart number (4-bytes) | // +----------------------+------------------------+-----+--------------------------+ // -// Note: Both the key offset and the value offset are relative to the start of -// the trie data chunk. To obtain the absolute offset, add the offset of the -// trie data chunk itself. +// Note: Both the key offset and the value offset are relative to the beginning +// of the trie data chunk. The chunk's base offset must be added to obtain the +// absolute position in the value section. // // Value section: // @@ -138,9 +140,12 @@ type trienodeHistory struct { // newTrienodeHistory constructs a trienode history with the provided trie nodes. func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory { - nodeList := make(map[common.Hash][]string) + nodeList := make(map[common.Hash][]string, len(nodes)) for owner, subset := range nodes { - keys := sort.StringSlice(slices.Collect(maps.Keys(subset))) + keys := make(sort.StringSlice, 0, len(subset)) + for k := range subset { + keys = append(keys, k) + } keys.Sort() nodeList[owner] = keys } @@ -157,17 +162,6 @@ func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, node } } -// sharedLen returns the length of the common prefix shared by a and b. -func sharedLen(a, b []byte) int { - n := min(len(a), len(b)) - for i := range n { - if a[i] != b[i] { - return i - } - } - return n -} - // typ implements the history interface, returning the historical data type held. func (h *trienodeHistory) typ() historyType { return typeTrienodeHistory @@ -175,11 +169,35 @@ func (h *trienodeHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *trienodeHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *trienodeHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, owner := range h.owners { - for _, path := range h.nodeList[owner] { - if !yield(newTrienodeIdent(owner, path)) { + var ( + scheme *indexScheme + paths = h.nodeList[owner] + indexes = make(map[string]map[uint16]struct{}) + ) + if owner == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + for _, leaf := range findLeafPaths(paths) { + chunks, ids := scheme.splitPath(leaf) + for i := 0; i < len(chunks); i++ { + if _, exists := indexes[chunks[i]]; !exists { + indexes[chunks[i]] = make(map[uint16]struct{}) + } + indexes[chunks[i]][ids[i]] = struct{}{} + } + } + for chunk, ids := range indexes { + elem := trienodeIndexElem{ + owner: owner, + path: chunk, + data: slices.Collect(maps.Keys(ids)), + } + if !yield(elem) { return } } @@ -207,17 +225,22 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { restarts []uint32 prefixLen int - internalKeyOffset uint32 // key offset for the trie internally - internalValOffset uint32 // value offset for the trie internally + internalKeyOffset uint32 // key offset within the trie data internally + internalValOffset uint32 // value offset within the trie data internally ) for i, path := range h.nodeList[owner] { key := []byte(path) + + // Track the internal key and value offsets at the beginning of the + // restart section. The absolute offsets within the key and value + // sections should first include the offset of the trie chunk itself + // stored in the header section. if i%trienodeDataBlockRestartLen == 0 { restarts = append(restarts, internalKeyOffset) restarts = append(restarts, internalValOffset) prefixLen = 0 } else { - prefixLen = sharedLen(prevKey, key) + prefixLen = commonPrefixLen(prevKey, key) } value := h.nodes[owner][path] @@ -256,18 +279,13 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { } // Fill the header section with the offsets of the key and value sections. - // Note that the key/value offsets are intentionally tracked *after* encoding - // them into their respective sections, ensuring each offset refers to the end - // position. For n trie chunks, n offset pairs are sufficient to uniquely locate - // the corresponding data. - headerSection.Write(owner.Bytes()) // 32 bytes - binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes - - // The offset to the value section is theoretically unnecessary, since the - // individual value offset is already tracked in the key section. However, - // we still keep it here for two reasons: - // - It's cheap to store (only 4 bytes for each trie). - // - It can be useful for decoding the trie data when key is not required (e.g., in hash mode). + // Note that key/value offsets are intentionally recorded *after* encoding + // into their respective sections, so each offset refers to an end position. + // For n trie chunks, n offset pairs are sufficient to uniquely locate each + // chunk's data. For example, [0, offset_0] defines the range of trie chunk 0, + // while [offset_{n-2}, offset_{n-1}] defines the range of trie chunk n-1. + headerSection.Write(owner.Bytes()) // 32 bytes + binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes binary.Write(&headerSection, binary.BigEndian, uint32(valueSection.Len())) // 4 bytes } return headerSection.Bytes(), keySection.Bytes(), valueSection.Bytes(), nil @@ -330,32 +348,68 @@ func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []ui }, owners, keyOffsets, valOffsets, nil } -func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) { - var ( - prevKey []byte - items int - keyOffsets []uint32 - valOffsets []uint32 +// decodeKeyEntry resolves a single entry from the key section starting from +// the specified offset. +func decodeKeyEntry(keySection []byte, offset int) (uint64, uint64, []byte, int, error) { + var byteRead int - keyOff int // the key offset within the single trie data - valOff int // the value offset within the single trie data + // Resolve the length of shared key + nShared, nn := binary.Uvarint(keySection[offset:]) // key length shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nShared at offset %d", offset) + } + byteRead += nn - keys []string - ) + // Resolve the length of unshared key + nUnshared, nn := binary.Uvarint(keySection[offset+byteRead:]) // key length not shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nUnshared at offset %d", offset+byteRead) + } + byteRead += nn + + // Resolve the length of value + nValue, nn := binary.Uvarint(keySection[offset+byteRead:]) // value length (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nValue at offset %d", offset+byteRead) + } + byteRead += nn + + // Validate that the values can fit in an int to prevent overflow on 32-bit systems + if nShared > uint64(math.MaxUint32) || nUnshared > uint64(math.MaxUint32) || nValue > uint64(math.MaxUint32) { + return 0, 0, nil, 0, errors.New("key/value size too large") + } + + // Resolve the unshared key + if offset+byteRead+int(nUnshared) > len(keySection) { + return 0, 0, nil, 0, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, offset+byteRead, len(keySection)) + } + unsharedKey := keySection[offset+byteRead : offset+byteRead+int(nUnshared)] + byteRead += int(nUnshared) + + return nShared, nValue, unsharedKey, byteRead, nil +} + +// decodeRestartTrailer resolves all the offsets recorded at the trailer. +func decodeRestartTrailer(keySection []byte) ([]uint32, []uint32, int, error) { // Decode the number of restart section if len(keySection) < 4 { - return nil, fmt.Errorf("key section too short, size: %d", len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, size: %d", len(keySection)) } nRestarts := binary.BigEndian.Uint32(keySection[len(keySection)-4:]) + // Decode the trailer + var ( + keyOffsets = make([]uint32, 0, int(nRestarts)) + valOffsets = make([]uint32, 0, int(nRestarts)) + ) if len(keySection) < int(8*nRestarts)+4 { - return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) } for i := range int(nRestarts) { o := len(keySection) - 4 - (int(nRestarts)-i)*8 keyOffset := binary.BigEndian.Uint32(keySection[o : o+4]) if i != 0 && keyOffset <= keyOffsets[i-1] { - return nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) + return nil, nil, 0, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) } keyOffsets = append(keyOffsets, keyOffset) @@ -363,84 +417,118 @@ func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]st // section have zero-size value. valOffset := binary.BigEndian.Uint32(keySection[o+4 : o+8]) if i != 0 && valOffset < valOffsets[i-1] { - return nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) + return nil, nil, 0, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) } valOffsets = append(valOffsets, valOffset) } - keyLimit := len(keySection) - 4 - int(nRestarts)*8 + keyLimit := len(keySection) - 4 - int(nRestarts)*8 // End of key data + return keyOffsets, valOffsets, keyLimit, nil +} +// decodeRestartSection resolves all entries in a restart section. The keyData +// contains the encoded keys for the section. +// +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within the restart section, the base value +// offset of the restart section itself should be added by the caller itself. +// What's more, this function should return `aborted == true` if the entry +// resolution should be terminated. +func decodeRestartSection(keyData []byte, onValue func(key []byte, start int, limit int) (bool, error)) error { + var ( + prevKey []byte + items int + + keyOff int // the key offset within the single trie data + valOff int // the value offset within the single trie data + ) // Decode data - for keyOff < keyLimit { - // Validate the key and value offsets within the single trie data chunk - if items%trienodeDataBlockRestartLen == 0 { - restartIndex := items / trienodeDataBlockRestartLen - if restartIndex >= len(keyOffsets) { - return nil, fmt.Errorf("restart index out of range: %d, available restarts: %d", restartIndex, len(keyOffsets)) - } - if keyOff != int(keyOffsets[restartIndex]) { - return nil, fmt.Errorf("key offset is not matched, recorded: %d, want: %d", keyOffsets[restartIndex], keyOff) - } - if valOff != int(valOffsets[restartIndex]) { - return nil, fmt.Errorf("value offset is not matched, recorded: %d, want: %d", valOffsets[restartIndex], valOff) - } + for keyOff < len(keyData) { + nShared, nValue, unsharedKey, nn, err := decodeKeyEntry(keyData, keyOff) + if err != nil { + return err } - // Resolve the entry from key section - nShared, nn := binary.Uvarint(keySection[keyOff:]) // key length shared (varint) keyOff += nn - nUnshared, nn := binary.Uvarint(keySection[keyOff:]) // key length not shared (varint) - keyOff += nn - nValue, nn := binary.Uvarint(keySection[keyOff:]) // value length (varint) - keyOff += nn - - // Resolve unshared key - if keyOff+int(nUnshared) > len(keySection) { - return nil, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, keyOff, len(keySection)) - } - unsharedKey := keySection[keyOff : keyOff+int(nUnshared)] - keyOff += int(nUnshared) // Assemble the full key var key []byte if items%trienodeDataBlockRestartLen == 0 { if nShared != 0 { - return nil, fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) + return fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) } key = unsharedKey } else { if int(nShared) > len(prevKey) { - return nil, fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) + return fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) } - key = append([]byte{}, prevKey[:nShared]...) - key = append(key, unsharedKey...) + key = make([]byte, int(nShared)+len(unsharedKey)) + copy(key[:nShared], prevKey[:nShared]) + copy(key[nShared:], unsharedKey) } if items != 0 && bytes.Compare(prevKey, key) >= 0 { - return nil, fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) + return fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) } prevKey = key - // Resolve value - if onValue != nil { - if err := onValue(key, valOff, valOff+int(nValue)); err != nil { - return nil, err - } + valEnd := valOff + int(nValue) + abort, err := onValue(key, valOff, valEnd) + if err != nil { + return err } - valOff += int(nValue) - + if abort { + return nil + } + valOff = valEnd items++ - keys = append(keys, string(key)) } - if keyOff != keyLimit { - return nil, fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, keyLimit) + if keyOff != len(keyData) { + return fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, len(keyData)) + } + return nil +} + +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within this trie chunk, the base value +// offset of the trie chunk itself should be added by the caller itself. +func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) error { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(keySection) + if err != nil { + return err + } + for i := 0; i < len(keyOffsets); i++ { + var keyData []byte + if i == len(keyOffsets)-1 { + keyData = keySection[keyOffsets[i]:keyLimit] + } else { + keyData = keySection[keyOffsets[i]:keyOffsets[i+1]] + } + err := decodeRestartSection(keyData, func(key []byte, start int, limit int) (bool, error) { + valStart := int(valOffsets[i]) + start + valLimit := int(valOffsets[i]) + limit + + // Possible in tests + if onValue == nil { + return false, nil + } + if err := onValue(key, valStart, valLimit); err != nil { + return false, err + } + return false, nil // abort=false + }) + if err != nil { + return err + } } - return keys, nil + return nil } func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, map[string][]byte, error) { var ( - offset int - nodes = make(map[string][]byte) + offset int + estimated = len(keySection) / 8 + nodes = make(map[string][]byte, estimated) + paths = make([]string, 0, estimated) ) - paths, err := decodeSingle(keySection, func(key []byte, start int, limit int) error { + err := decodeSingle(keySection, func(key []byte, start int, limit int) error { if start != offset { return fmt.Errorf("gapped value section offset: %d, want: %d", start, offset) } @@ -451,7 +539,9 @@ func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, ma if start > len(valueSection) || limit > len(valueSection) { return fmt.Errorf("value section out of range: start: %d, limit: %d, size: %d", start, limit, len(valueSection)) } - nodes[string(key)] = valueSection[start:limit] + strkey := string(key) + paths = append(paths, strkey) + nodes[strkey] = valueSection[start:limit] offset = limit return nil @@ -477,7 +567,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection h.nodes = make(map[common.Hash]map[string][]byte) for i := range len(owners) { - // Resolve the boundary of key section + // Resolve the boundary of the key section, each offset referring + // to the end position of this trie chunk. var keyStart, keyLimit uint32 if i != 0 { keyStart = keyOffsets[i-1] @@ -487,7 +578,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection return fmt.Errorf("invalid key offsets: keyStart: %d, keyLimit: %d, size: %d", keyStart, keyLimit, len(keySection)) } - // Resolve the boundary of value section + // Resolve the boundary of the value section, each offset referring + // to the end position of this trie chunk. var valStart, valLimit uint32 if i != 0 { valStart = valueOffsets[i-1] @@ -517,14 +609,11 @@ func (ir iRange) len() uint32 { return ir.limit - ir.start } -// singleTrienodeHistoryReader provides read access to a single trie within the -// trienode history. It stores an offset to the trie's position in the history, -// along with a set of per-node offsets that can be resolved on demand. type singleTrienodeHistoryReader struct { - id uint64 - reader ethdb.AncientReader - valueRange iRange // value range within the global value section - valueInternalOffsets map[string]iRange // value offset within the single trie data + id uint64 + reader ethdb.AncientReader + keyData []byte + valueRange iRange } func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRange iRange, valueRange iRange) (*singleTrienodeHistoryReader, error) { @@ -532,121 +621,173 @@ func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRa if err != nil { return nil, err } - valueOffsets := make(map[string]iRange) - _, err = decodeSingle(keyData, func(key []byte, start int, limit int) error { - valueOffsets[string(key)] = iRange{ - start: uint32(start), - limit: uint32(limit), - } - return nil + return &singleTrienodeHistoryReader{ + id: id, + reader: reader, + keyData: keyData, + valueRange: valueRange, + }, nil +} + +// searchSingle searches for a specific trie node identified by the provided +// key within a single trie node chunk. +// +// It returns the node value's offset range (start and limit) within the +// trie node data. An error is returned if the node cannot be found. +func (sr *singleTrienodeHistoryReader) searchSingle(key []byte) (int, int, bool, error) { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(sr.keyData) + if err != nil { + return 0, 0, false, err + } + // Binary search against the boundary keys for each restart section + var ( + boundFind bool + boundValueLen uint64 + ) + pos := sort.Search(len(keyOffsets), func(i int) bool { + _, nValue, dkey, _, derr := decodeKeyEntry(sr.keyData[keyOffsets[i]:], 0) + if derr != nil { + err = derr + return false + } + n := bytes.Compare(key, dkey) + if n == 0 { + boundFind = true + boundValueLen = nValue + } + return n <= 0 }) if err != nil { - return nil, err + return 0, 0, false, err } - return &singleTrienodeHistoryReader{ - id: id, - reader: reader, - valueRange: valueRange, - valueInternalOffsets: valueOffsets, - }, nil + // The node is found as the boundary of restart section + if boundFind { + start := valOffsets[pos] + limit := valOffsets[pos] + uint32(boundValueLen) + return int(start), int(limit), true, nil + } + // The node is not found as all others have larger key than the target + if pos == 0 { + return 0, 0, false, nil + } + // Search the target node within the restart section + var keyData []byte + if pos == len(keyOffsets) { + keyData = sr.keyData[keyOffsets[pos-1]:keyLimit] // last section + } else { + keyData = sr.keyData[keyOffsets[pos-1]:keyOffsets[pos]] // non-last section + } + var ( + nStart int + nLimit int + found bool + ) + err = decodeRestartSection(keyData, func(ikey []byte, start, limit int) (bool, error) { + if bytes.Equal(key, ikey) { + nStart = int(valOffsets[pos-1]) + start + nLimit = int(valOffsets[pos-1]) + limit + found = true + return true, nil // abort = true + } + return false, nil // abort = false + }) + if err != nil { + return 0, 0, false, err + } + if !found { + return 0, 0, false, nil + } + return nStart, nLimit, true, nil } // read retrieves the trie node data with the provided node path. -func (sr *singleTrienodeHistoryReader) read(path string) ([]byte, error) { - offset, exists := sr.valueInternalOffsets[path] - if !exists { - return nil, fmt.Errorf("trienode %v not found", []byte(path)) +func (sr *singleTrienodeHistoryReader) read(key []byte) ([]byte, bool, error) { + start, limit, found, err := sr.searchSingle(key) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + valStart := uint64(start) + uint64(sr.valueRange.start) + valLen := uint64(limit - start) + value, err := rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, valStart, valLen) + if err != nil { + return nil, false, err } - return rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, uint64(sr.valueRange.start+offset.start), uint64(offset.len())) + return value, true, nil } // trienodeHistoryReader provides read access to node data in the trie node history. // It resolves data from the underlying ancient store only when needed, minimizing // I/O overhead. type trienodeHistoryReader struct { - id uint64 // ID of the associated trienode history - reader ethdb.AncientReader // Database reader of ancient store - keyRanges map[common.Hash]iRange // Key ranges identifying trie chunks - valRanges map[common.Hash]iRange // Value ranges identifying trie chunks - iReaders map[common.Hash]*singleTrienodeHistoryReader // readers for each individual trie chunk + id uint64 // ID of the associated trienode history + reader ethdb.AncientReader // Database reader of ancient store } // newTrienodeHistoryReader constructs the reader for specific trienode history. -func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeHistoryReader, error) { - r := &trienodeHistoryReader{ - id: id, - reader: reader, - keyRanges: make(map[common.Hash]iRange), - valRanges: make(map[common.Hash]iRange), - iReaders: make(map[common.Hash]*singleTrienodeHistoryReader), - } - if err := r.decodeHeader(); err != nil { - return nil, err +func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) *trienodeHistoryReader { + return &trienodeHistoryReader{ + id: id, + reader: reader, } - return r, nil } // decodeHeader decodes the header section of trienode history. -func (r *trienodeHistoryReader) decodeHeader() error { +func (r *trienodeHistoryReader) decodeHeader(owner common.Hash) (iRange, iRange, bool, error) { header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id) if err != nil { - return err + return iRange{}, iRange{}, false, err } _, owners, keyOffsets, valOffsets, err := decodeHeader(header) if err != nil { - return err + return iRange{}, iRange{}, false, err } - for i, owner := range owners { - // Decode the key range for this trie chunk - var keyStart uint32 - if i != 0 { - keyStart = keyOffsets[i-1] - } - r.keyRanges[owner] = iRange{ - start: keyStart, - limit: keyOffsets[i], - } + pos := sort.Search(len(owners), func(i int) bool { + return owner.Cmp(owners[i]) <= 0 + }) + if pos == len(owners) || owners[pos] != owner { + return iRange{}, iRange{}, false, nil + } + var keyRange iRange + if pos != 0 { + keyRange.start = keyOffsets[pos-1] + } + keyRange.limit = keyOffsets[pos] - // Decode the value range for this trie chunk - var valStart uint32 - if i != 0 { - valStart = valOffsets[i-1] - } - r.valRanges[owner] = iRange{ - start: valStart, - limit: valOffsets[i], - } + var valRange iRange + if pos != 0 { + valRange.start = valOffsets[pos-1] } - return nil + valRange.limit = valOffsets[pos] + return keyRange, valRange, true, nil } // read retrieves the trie node data with the provided TrieID and node path. -func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, error) { - ir, ok := r.iReaders[owner] - if !ok { - keyRange, exists := r.keyRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - valRange, exists := r.valRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - var err error - ir, err = newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) - if err != nil { - return nil, err - } - r.iReaders[owner] = ir +func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, bool, error) { + keyRange, valRange, found, err := r.decodeHeader(owner) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + ir, err := newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) + if err != nil { + return nil, false, err } - return ir.read(path) + return ir.read([]byte(path)) } // writeTrienodeHistory persists the trienode history associated with the given diff layer. -// nolint:unused -func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { +func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer, rate uint32) error { start := time.Now() - h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin) + nodes, err := dl.nodes.encodeNodeHistory(dl.root, rate) + if err != nil { + return err + } + h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, nodes) header, keySection, valueSection, err := h.encode() if err != nil { return err @@ -670,7 +811,6 @@ func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { } // readTrienodeMetadata resolves the metadata of the specified trienode history. -// nolint:unused func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) { header, err := rawdb.ReadTrienodeHistoryHeader(reader, id) if err != nil { diff --git a/triedb/pathdb/history_trienode_test.go b/triedb/pathdb/history_trienode_test.go index d6b80f61f5..05278c30b1 100644 --- a/triedb/pathdb/history_trienode_test.go +++ b/triedb/pathdb/history_trienode_test.go @@ -19,6 +19,7 @@ package pathdb import ( "bytes" "encoding/binary" + "fmt" "math/rand" "reflect" "testing" @@ -137,14 +138,11 @@ func TestTrienodeHistoryReader(t *testing.T) { } } for i, h := range hs { - tr, err := newTrienodeHistoryReader(uint64(i+1), freezer) - if err != nil { - t.Fatalf("Failed to construct the history reader: %v", err) - } + tr := newTrienodeHistoryReader(uint64(i+1), freezer) for _, owner := range h.owners { nodes := h.nodes[owner] for key, value := range nodes { - blob, err := tr.read(owner, key) + blob, _, err := tr.read(owner, key) if err != nil { t.Fatalf("Failed to read trienode history: %v", err) } @@ -417,23 +415,23 @@ func TestTrienodeHistoryReaderNonExistentPath(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Try to read a non-existent path - _, err = tr.read(testrand.Hash(), "nonexistent") - if err == nil { - t.Fatal("Expected error for non-existent trie owner") + var ( + err error + found bool + ) + _, found, err = tr.read(testrand.Hash(), "nonexistent") + if found || err != nil { + t.Fatal("Expected not found for non-existent trie owner") } // Try to read from existing owner but non-existent path owner := h.owners[0] - _, err = tr.read(owner, "nonexistent-path") - if err == nil { - t.Fatal("Expected error for non-existent path") + _, found, err = tr.read(owner, "nonexistent-path") + if found || err != nil { + t.Fatal("Expected not found for non-existent path") } } @@ -457,23 +455,19 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "nil1") - if err != nil { + data1, found, err := tr.read(owner, "nil1") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data1) != 0 { t.Fatal("Expected nil data for nil value") } - data2, err := tr.read(owner, "nil2") - if err != nil { + data2, found, err := tr.read(owner, "nil2") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data2) != 0 { @@ -481,8 +475,8 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { } // Test reading non-nil value - data3, err := tr.read(owner, "data1") - if err != nil { + data3, found, err := tr.read(owner, "data1") + if err != nil || !found { t.Fatalf("Failed to read non-nil value: %v", err) } if !bytes.Equal(data3, []byte("some data")) { @@ -498,7 +492,7 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { // Add some nil values nodes[owner][""] = []byte("some data") - nodes[owner]["data1"] = []byte("some data") + nodes[owner]["data1"] = []byte("some data1") h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) @@ -509,14 +503,10 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "") + data1, _, err := tr.read(owner, "") if err != nil { t.Fatalf("Failed to read nil value: %v", err) } @@ -525,63 +515,17 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { } // Test reading non-nil value - data2, err := tr.read(owner, "data1") + data2, _, err := tr.read(owner, "data1") if err != nil { t.Fatalf("Failed to read non-nil value: %v", err) } - if !bytes.Equal(data2, []byte("some data")) { + if !bytes.Equal(data2, []byte("some data1")) { t.Fatal("Data mismatch for non-nil key") } } -// TestTrienodeHistoryReaderIterator tests the iterator functionality -func TestTrienodeHistoryReaderIterator(t *testing.T) { - h := makeTrienodeHistory() - - // Count expected entries - expectedCount := 0 - expectedNodes := make(map[stateIdent]bool) - for owner, nodeList := range h.nodeList { - expectedCount += len(nodeList) - for _, node := range nodeList { - expectedNodes[stateIdent{ - typ: typeTrienode, - addressHash: owner, - path: node, - }] = true - } - } - - // Test the iterator - actualCount := 0 - for x := range h.forEach() { - _ = x - actualCount++ - } - if actualCount != expectedCount { - t.Fatalf("Iterator count mismatch: expected %d, got %d", expectedCount, actualCount) - } - - // Test that iterator yields expected state identifiers - seen := make(map[stateIdent]bool) - for ident := range h.forEach() { - if ident.typ != typeTrienode { - t.Fatal("Iterator should only yield trienode history identifiers") - } - key := stateIdent{typ: ident.typ, addressHash: ident.addressHash, path: ident.path} - if seen[key] { - t.Fatal("Iterator yielded duplicate identifier") - } - seen[key] = true - - if !expectedNodes[key] { - t.Fatalf("Unexpected yielded identifier %v", key) - } - } -} - -// TestSharedLen tests the sharedLen helper function -func TestSharedLen(t *testing.T) { +// TestCommonPrefixLen tests the commonPrefixLen helper function +func TestCommonPrefixLen(t *testing.T) { tests := []struct { a, b []byte expected int @@ -610,13 +554,13 @@ func TestSharedLen(t *testing.T) { } for i, test := range tests { - result := sharedLen(test.a, test.b) + result := commonPrefixLen(test.a, test.b) if result != test.expected { t.Errorf("Test %d: sharedLen(%q, %q) = %d, expected %d", i, test.a, test.b, result, test.expected) } // Test commutativity - resultReverse := sharedLen(test.b, test.a) + resultReverse := commonPrefixLen(test.b, test.a) if result != resultReverse { t.Errorf("Test %d: sharedLen is not commutative: sharedLen(a,b)=%d, sharedLen(b,a)=%d", i, result, resultReverse) @@ -678,14 +622,14 @@ func TestDecodeSingleCorruptedData(t *testing.T) { _, keySection, _, _ := h.encode() // Test with empty key section - _, err := decodeSingle([]byte{}, nil) + err := decodeSingle([]byte{}, nil) if err == nil { t.Fatal("Expected error for empty key section") } // Test with key section too small for trailer if len(keySection) > 0 { - _, err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer + err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer if err == nil { t.Fatal("Expected error for key section too small for trailer") } @@ -694,8 +638,11 @@ func TestDecodeSingleCorruptedData(t *testing.T) { // Test with corrupted varint in key section corrupted := make([]byte, len(keySection)) copy(corrupted, keySection) - corrupted[5] = 0xFF // Corrupt varint - _, err = decodeSingle(corrupted, nil) + // Fill first 10 bytes with 0xFF to create a varint overflow (>64 bits) + for i := range 10 { + corrupted[i] = 0xFF + } + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for corrupted varint") } @@ -705,7 +652,7 @@ func TestDecodeSingleCorruptedData(t *testing.T) { copy(corrupted, keySection) // Set restart count to something too large binary.BigEndian.PutUint32(corrupted[len(corrupted)-4:], 10000) - _, err = decodeSingle(corrupted, nil) + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for invalid restart count") } @@ -734,3 +681,57 @@ func testEncodeDecode(t *testing.T, h *trienodeHistory) { t.Fatal("Trienode content mismatch") } } + +func TestSearchSingle(t *testing.T) { + nodes := make(map[common.Hash]map[string][]byte) + ownerA, ownerB := testrand.Hash(), testrand.Hash() + nodes[ownerA] = make(map[string][]byte) + nodes[ownerB] = make(map[string][]byte) + + for i := 0; i < trienodeDataBlockRestartLen*2; i++ { + nodes[ownerA][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + nodes[ownerB][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + } + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) + + var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false) + defer freezer.Close() + + header, keySection, valueSection, _ := h.encode() + if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { + t.Fatalf("Failed to write trienode history: %v", err) + } + tr := newTrienodeHistoryReader(1, freezer) + + // Test reading non-existent entry + keys := []string{ + "0", + "2", + "30", + "32", + "64", + "1000", + } + for _, key := range keys { + _, found, err := tr.read(ownerA, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + _, found, err = tr.read(ownerB, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + } + + for owner, subnodes := range nodes { + for key, value := range subnodes { + got, found, err := tr.read(owner, key) + if err != nil || !found { + t.Fatal("Failed to read trienode") + } + if bytes.Compare(got, value) != 0 { + t.Fatalf("Unexpected value for key %v, got %v, expected %v", []byte(key), got, value) + } + } + } +} diff --git a/triedb/pathdb/history_trienode_utils.go b/triedb/pathdb/history_trienode_utils.go new file mode 100644 index 0000000000..11107494bb --- /dev/null +++ b/triedb/pathdb/history_trienode_utils.go @@ -0,0 +1,344 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "encoding/binary" + "fmt" + "math/bits" + "slices" + "strings" +) + +// commonPrefixLen returns the length of the common prefix shared by a and b. +func commonPrefixLen(a, b []byte) int { + n := min(len(a), len(b)) + for i := range n { + if a[i] != b[i] { + return i + } + } + return n +} + +// findLeafPaths scans a lexicographically sorted list of paths and returns +// the subset of paths that represent leaves. +// +// A path is considered a leaf if: +// - it is the last element in the list, or +// - the next path does not have the current path as its prefix. +// +// In other words, a leaf is a path that has no children extending it. +// +// Example: +// +// Input: ["a", "ab", "abc", "b", "ba"] +// Output: ["abc", "ba"] +// +// The input must be sorted; otherwise the result is undefined. +func findLeafPaths(paths []string) []string { + var leaves []string + for i := 0; i < len(paths); i++ { + if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { + leaves = append(leaves, paths[i]) + } + } + return leaves +} + +// hexPathNodeID computes a numeric node ID from the given path. The path is +// interpreted as a sequence of base-16 digits, where each byte of the input +// is treated as one hexadecimal digit in a big-endian number. +// +// The resulting node ID is constructed as: +// +// ID = 1 + 16 + 16^2 + ... + 16^(n-1) + value +// +// where n is the number of bytes in the path, and `value` is the base-16 +// interpretation of the byte sequence. +// +// The offset (1 + 16 + 16^2 + ... + 16^(n-1)) ensures that all IDs of shorter +// paths occupy a lower numeric range, preserving lexicographic ordering between +// differently-length paths. +// +// The numeric node ID is represented by the uint16 with the assumption the length +// of path won't be greater than 3. +func hexPathNodeID(path string) uint16 { + var ( + offset = uint16(0) + pow = uint16(1) + value = uint16(0) + bytes = []byte(path) + ) + for i := 0; i < len(bytes); i++ { + offset += pow + pow *= 16 + } + for i := 0; i < len(bytes); i++ { + value = value*16 + uint16(bytes[i]) + } + return offset + value +} + +// bitmapSize computes the number of bytes required for the marker bitmap +// corresponding to the remaining portion of a path after a cut point. +// The marker is a bitmap where each bit represents the presence of a +// possible element in the remaining path segment. +func bitmapSize(levels int) int { + // Compute: total = 1 + 16 + 16^2 + ... + 16^(segLen-1) + var ( + bits = 0 + pow = 1 + ) + for i := 0; i < levels; i++ { + bits += pow + pow *= 16 + } + // A small adjustment is applied to exclude the root element of this path + // segment, since any existing element would already imply the mutation of + // the root element. This trick can save us 1 byte for each bitmap which is + // non-trivial. + bits -= 1 + return bits / 8 +} + +// indexScheme defines how trie nodes are split into chunks and index them +// at chunk level. +// +// skipRoot indicates whether the root node should be excluded from indexing. +// cutPoints specifies the key length of chunks (in nibbles) extracted from +// each path. +type indexScheme struct { + // skipRoot indicates whether the root node should be excluded from indexing. + // In the account trie, the root is mutated on every state transition, so + // indexing it provides no value. + skipRoot bool + + // cutPoints defines the key lengths of chunks at different positions. + // A single trie node path may span multiple chunks vertically. + cutPoints []int + + // bitmaps specifies the required bitmap size for each chunk. The key is the + // chunk key length, and the value is the corresponding bitmap size. + bitmaps map[int]int +} + +var ( + // Account trie is split into chunks like this: + // + // - root node is excluded from indexing + // - nodes at level1 to level2 are grouped as 16 chunks + // - all other nodes are grouped 3 levels per chunk + // + // Level1 [0] ... [f] 16 chunks + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // For the chunks at level1, there are 17 nodes per chunk. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // + // For the non-level1 chunks, there are 273 nodes per chunk, + // regardless of the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + accountIndexScheme = newIndexScheme(true) + + // Storage trie is split into chunks like this: (3 levels per chunk) + // + // Level0 [ ROOT ] 1 chunk + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // Within each chunk, there are 273 nodes in total, regardless of + // the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + storageIndexScheme = newIndexScheme(false) +) + +// newIndexScheme initializes the index scheme. +func newIndexScheme(skipRoot bool) *indexScheme { + var ( + cuts []int + bitmaps = make(map[int]int) + ) + for v := 0; v <= 64; v += 3 { + var ( + levels int + length int + ) + if v == 0 && skipRoot { + length = 1 + levels = 2 + } else { + length = v + levels = 3 + } + cuts = append(cuts, length) + bitmaps[length] = bitmapSize(levels) + } + return &indexScheme{ + skipRoot: skipRoot, + cutPoints: cuts, + bitmaps: bitmaps, + } +} + +// getBitmapSize returns the required bytes for bitmap with chunk's position. +func (s *indexScheme) getBitmapSize(pathLen int) int { + return s.bitmaps[pathLen] +} + +// chunkSpan returns how many chunks should be spanned with the given path. +func (s *indexScheme) chunkSpan(length int) int { + var n int + for _, cut := range s.cutPoints { + if length >= cut { + n++ + continue + } + } + return n +} + +// splitPath applies the indexScheme to the given path and returns two lists: +// +// - chunkIDs: the progressive chunk IDs cuts defined by the scheme +// - innerIDs: the computed node ID for the path segment following each cut +// +// The scheme defines a set of cut points that partition the path. For each cut: +// +// - chunkIDs[i] is path[:cutPoints[i]] +// - innerIDs[i] is the node ID of the segment path[cutPoints[i] : nextCut-1] +func (s *indexScheme) splitPath(path string) ([]string, []uint16) { + // Special case: the root node of the account trie is mutated in every + // state transition, so its mutation records can be ignored. + n := len(path) + if n == 0 && s.skipRoot { + return nil, nil + } + var ( + // Determine how many chunks are spanned by the path + chunks = s.chunkSpan(n) + chunkIDs = make([]string, 0, chunks) + nodeIDs = make([]uint16, 0, chunks) + ) + for i := 0; i < chunks; i++ { + position := s.cutPoints[i] + chunkIDs = append(chunkIDs, path[:position]) + + var limit int + if i != chunks-1 { + limit = s.cutPoints[i+1] - 1 + } else { + limit = len(path) + } + nodeIDs = append(nodeIDs, hexPathNodeID(path[position:limit])) + } + return chunkIDs, nodeIDs +} + +// splitPathLast returns the path prefix of the deepest chunk spanned by the +// given path, along with its corresponding internal node ID. If the path +// spans no chunks, it returns an empty prefix and 0. +// +// nolint:unused +func (s *indexScheme) splitPathLast(path string) (string, uint16) { + chunkIDs, nodeIDs := s.splitPath(path) + if len(chunkIDs) == 0 { + return "", 0 + } + n := len(chunkIDs) + return chunkIDs[n-1], nodeIDs[n-1] +} + +// encodeIDs sorts the given list of uint16 IDs and encodes them into a +// compact byte slice using variable-length unsigned integer encoding. +func encodeIDs(ids []uint16) []byte { + slices.Sort(ids) + buf := make([]byte, 0, len(ids)) + for _, id := range ids { + buf = binary.AppendUvarint(buf, uint64(id)) + } + return buf +} + +// decodeIDs decodes a sequence of variable-length encoded uint16 IDs from the +// given byte slice and returns them as a set. +// +// Returns an error if the input buffer does not contain a complete Uvarint value. +func decodeIDs(buf []byte) ([]uint16, error) { + var res []uint16 + for len(buf) > 0 { + id, n := binary.Uvarint(buf) + if n <= 0 { + return nil, fmt.Errorf("too short for decoding node id, %v", buf) + } + buf = buf[n:] + res = append(res, uint16(id)) + } + return res, nil +} + +// isAncestor reports whether node x is the ancestor of node y. +func isAncestor(x, y uint16) bool { + for y > x { + y = (y - 1) / 16 // parentID(y) = (y - 1) / 16 + if y == x { + return true + } + } + return false +} + +// isBitSet reports whether the bit at `index` in the byte slice `b` is set. +func isBitSet(b []byte, index int) bool { + return b[index/8]&(1<<(7-index%8)) != 0 +} + +// setBit sets the bit at `index` in the byte slice `b` to 1. +func setBit(b []byte, index int) { + b[index/8] |= 1 << (7 - index%8) +} + +// bitPosTwoBytes returns the positions of set bits in a 2-byte bitmap. +// +// The bitmap is interpreted as a big-endian uint16. Bit positions are +// numbered from 0 to 15, where position 0 corresponds to the most +// significant bit of b[0], and position 15 corresponds to the least +// significant bit of b[1]. +func bitPosTwoBytes(b []byte) []int { + if len(b) != 2 { + panic("expect 2 bytes") + } + var ( + pos []int + mask = binary.BigEndian.Uint16(b) + ) + for mask != 0 { + p := bits.LeadingZeros16(mask) + pos = append(pos, p) + mask &^= 1 << (15 - p) + } + return pos +} diff --git a/triedb/pathdb/history_trienode_utils_test.go b/triedb/pathdb/history_trienode_utils_test.go new file mode 100644 index 0000000000..32bd91166d --- /dev/null +++ b/triedb/pathdb/history_trienode_utils_test.go @@ -0,0 +1,584 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "reflect" + "testing" +) + +func TestHexPathNodeID(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + exp uint16 + }{ + { + input: "", + exp: 0, + }, + { + input: string([]byte{0x0}), + exp: 1, + }, + { + input: string([]byte{0xf}), + exp: 16, + }, + { + input: string([]byte{0x0, 0x0}), + exp: 17, + }, + { + input: string([]byte{0x0, 0xf}), + exp: 32, + }, + { + input: string([]byte{0x1, 0x0}), + exp: 33, + }, + { + input: string([]byte{0x1, 0xf}), + exp: 48, + }, + { + input: string([]byte{0xf, 0xf}), + exp: 272, + }, + { + input: string([]byte{0xf, 0xf, 0xf}), + exp: 4368, + }, + } + for _, suite := range suites { + got := hexPathNodeID(suite.input) + if got != suite.exp { + t.Fatalf("Unexpected node ID for %v: got %d, want %d", suite.input, got, suite.exp) + } + } +} + +func TestFindLeafPaths(t *testing.T) { + t.Parallel() + + tests := []struct { + input []string + expect []string + }{ + { + input: nil, + expect: nil, + }, + { + input: []string{"a"}, + expect: []string{"a"}, + }, + { + input: []string{"", "0", "00", "01", "1"}, + expect: []string{ + "00", + "01", + "1", + }, + }, + { + input: []string{"10", "100", "11", "2"}, + expect: []string{ + "100", + "11", + "2", + }, + }, + { + input: []string{"10", "100000000", "11", "111111111", "2"}, + expect: []string{ + "100000000", + "111111111", + "2", + }, + }, + } + for _, test := range tests { + res := findLeafPaths(test.input) + if !reflect.DeepEqual(res, test.expect) { + t.Fatalf("Unexpected result: %v, expected %v", res, test.expect) + } + } +} + +func TestSplitAccountPath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", nil, nil, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{0x1}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 0, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 16, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 16, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, 0, + }, + }, + } + for _, suite := range suites { + prefix, id := accountIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Unexpected prefix for %v: got %v, want %v", suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Unexpected ID for %v: got %v, want %v", suite.input, id, suite.expID) + } + } +} + +func TestSplitStoragePath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", + []string{ + string([]byte{}), + }, + []uint16{ + 0, + }, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 16, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 17, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 18, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 32, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 272, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, 0, + }, + }, + } + for i, suite := range suites { + prefix, id := storageIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Test %d, unexpected prefix for %v: got %v, want %v", i, suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Test %d, unexpected ID for %v: got %v, want %v", i, suite.input, id, suite.expID) + } + } +} + +func TestIsAncestor(t *testing.T) { + suites := []struct { + x, y uint16 + want bool + }{ + {0, 1, true}, + {0, 16, true}, + {0, 17, true}, + {0, 272, true}, + + {1, 0, false}, + {1, 2, false}, + {1, 17, true}, + {1, 18, true}, + {17, 273, true}, + {1, 1, false}, + } + for _, tc := range suites { + result := isAncestor(tc.x, tc.y) + if result != tc.want { + t.Fatalf("isAncestor(%d, %d) = %v, want %v", tc.x, tc.y, result, tc.want) + } + } +} + +func TestBitmapSet(t *testing.T) { + suites := []struct { + index int + expect []byte + }{ + { + 0, []byte{0b10000000, 0x0}, + }, + { + 1, []byte{0b01000000, 0x0}, + }, + { + 7, []byte{0b00000001, 0x0}, + }, + { + 8, []byte{0b00000000, 0b10000000}, + }, + { + 15, []byte{0b00000000, 0b00000001}, + }, + } + for _, tc := range suites { + var buf [2]byte + setBit(buf[:], tc.index) + + if !bytes.Equal(buf[:], tc.expect) { + t.Fatalf("bitmap = %v, want %v", buf, tc.expect) + } + if !isBitSet(buf[:], tc.index) { + t.Fatal("bit is not set") + } + } +} + +func TestBitPositions(t *testing.T) { + suites := []struct { + input []byte + expect []int + }{ + { + []byte{0b10000000, 0x0}, []int{0}, + }, + { + []byte{0b01000000, 0x0}, []int{1}, + }, + { + []byte{0b00000001, 0x0}, []int{7}, + }, + { + []byte{0b00000000, 0b10000000}, []int{8}, + }, + { + []byte{0b00000000, 0b00000001}, []int{15}, + }, + { + []byte{0b10000000, 0b00000001}, []int{0, 15}, + }, + { + []byte{0b10000001, 0b00000001}, []int{0, 7, 15}, + }, + { + []byte{0b10000001, 0b10000001}, []int{0, 7, 8, 15}, + }, + { + []byte{0b11111111, 0b11111111}, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }, + { + []byte{0x0, 0x0}, nil, + }, + } + for _, tc := range suites { + got := bitPosTwoBytes(tc.input) + if !reflect.DeepEqual(got, tc.expect) { + t.Fatalf("Unexpected position set, want: %v, got: %v", tc.expect, got) + } + } +} diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 02bdef5d34..efcc3f2549 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -338,10 +338,8 @@ func (db *Database) Journal(root common.Hash) error { // but the ancient store is not properly closed, resulting in recent writes // being lost. After a restart, the ancient store would then be misaligned // with the disk layer, causing data corruption. - if db.stateFreezer != nil { - if err := db.stateFreezer.SyncAncient(); err != nil { - return err - } + if err := syncHistory(db.stateFreezer, db.trienodeFreezer); err != nil { + return err } // Store the journal into the database and return var ( diff --git a/triedb/pathdb/lookup.go b/triedb/pathdb/lookup.go index 8b092730f8..719546f410 100644 --- a/triedb/pathdb/lookup.go +++ b/triedb/pathdb/lookup.go @@ -33,6 +33,13 @@ func storageKey(accountHash common.Hash, slotHash common.Hash) [64]byte { return key } +// storageKeySlice returns a key for uniquely identifying the storage slot in +// the slice format. +func storageKeySlice(accountHash common.Hash, slotHash common.Hash) []byte { + key := storageKey(accountHash, slotHash) + return key[:] +} + // lookup is an internal structure used to efficiently determine the layer in // which a state entry resides. type lookup struct { diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 31c40053fc..a0a626f9b5 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -73,11 +73,8 @@ var ( stateHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/data", nil) stateHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/index", nil) - //nolint:unused - trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) - //nolint:unused - trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) - //nolint:unused + trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) + trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil) stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil) @@ -88,8 +85,9 @@ var ( lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) - historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) - historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) + historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalTrienodeReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/reads", nil) ) // Metrics in generation diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index c6f9e7aece..62c72c1953 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -14,12 +14,14 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// nolint:unused package pathdb import ( "bytes" "errors" "fmt" + "hash/fnv" "io" "maps" @@ -30,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -424,3 +427,265 @@ func (s *nodeSetWithOrigin) decode(r *rlp.Stream) error { s.computeSize() return nil } + +// encodeNodeCompressed encodes the trie node differences between two consecutive +// versions into byte stream. The format is as below: +// +// - metadata byte layout (1 byte): +// +// ┌──── Bits (from MSB to LSB) ───┐ +// │ 7 │ 6 │ 5 │ 4 │ 3 │ 2 │ 1 │ 0 │ +// └───────────────────────────────┘ +// │ │ │ │ │ │ │ └─ FlagA: set if value is encoded in compressed format +// │ │ │ │ │ │ └───── FlagB: set if no extended bitmap is present after the metadata byte +// │ │ │ │ │ └───────── FlagC: bitmap for node (only used when flagB == 1) +// │ │ │ │ └───────────── FlagD: bitmap for node (only used when flagB == 1) +// │ │ │ └───────────────── FlagE: reserved (marks the presence of the 16th child in a full node) +// │ │ └───────────────────── FlagF: reserved +// │ └───────────────────────── FlagG: reserved +// └───────────────────────────── FlagH: reserved +// +// Note: +// - If flagB is 1, the node refers to a shortNode; +// - flagC indicates whether the key of the shortNode is recorded. +// - flagD indicates whether the value of the shortNode is recorded. +// +// - If flagB is 0, the node refers to a fullNode; +// - each bit in extended bitmap indicates whether the corresponding +// child have been modified. +// +// Example: +// +// 0b_0000_1011 +// +// Bit0=1, Bit1=1 -> node in compressed format, no extended bitmap +// Bit2=0, Bit3=1 -> the key of a short node is not stored; its value is stored. +// +// - 2 bytes extended bitmap (only if the flagB in metadata is 0), each bit +// represents a corresponding child; +// +// - concatenation of original value of modified children along with its size; +func encodeNodeCompressed(addExtension bool, elements [][]byte, indices []int) []byte { + var ( + enc []byte + flag = byte(1) // The compression format indicator + ) + // Pre-allocate the byte slice for the node encoder + size := 1 + if addExtension { + size += 2 + } + for _, element := range elements { + size += len(element) + 1 + } + enc = make([]byte, 0, size) + + if !addExtension { + flag |= 2 // The embedded bitmap indicator + + // Embedded bitmap + for _, pos := range indices { + flag |= 1 << (pos + 2) + } + enc = append(enc, flag) + } else { + // Extended bitmap + bitmap := make([]byte, 2) // bitmaps for at most 16 children + for _, pos := range indices { + // Children[16] is only theoretically possible in the Merkle-Patricia-trie, + // in practice this field is never used in the Ethereum case. If it occurs, + // use the FlagE for marking the presence. + if pos >= 16 { + log.Warn("Unexpected 16th child encountered in a full node") + flag |= 1 << 4 // Use the reserved flagE + continue + } + setBit(bitmap, pos) + } + enc = append(enc, flag) + enc = append(enc, bitmap...) + } + for _, element := range elements { + enc = append(enc, byte(len(element))) // 1 byte is sufficient for element size + enc = append(enc, element...) + } + return enc +} + +// encodeNodeFull encodes the full trie node value into byte stream. The format is +// as below: +// +// - metadata byte layout (1 byte): 0b0 +// - node value +// +// TODO(rjl493456442) it's not allocation efficient, please improve it. +func encodeNodeFull(value []byte) []byte { + enc := make([]byte, len(value)+1) + copy(enc[1:], value) + return enc +} + +// decodeNodeCompressed decodes the byte stream of compressed trie node +// back to the original elements and their indices. +// +// It assumes the byte stream contains a compressed format node. +func decodeNodeCompressed(data []byte) ([][]byte, []int, error) { + if len(data) < 1 { + return nil, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag&byte(1) == 0 { + return nil, nil, errors.New("invalid data: full node value") + } + noExtend := flag&byte(2) != 0 + + // Reconstruct indices from bitmap + var indices []int + if noExtend { + if flag&byte(4) != 0 { // flagC + indices = append(indices, 0) + } + if flag&byte(8) != 0 { // flagD + indices = append(indices, 1) + } + data = data[1:] + } else { + if len(data) < 3 { + return nil, nil, errors.New("invalid data: too short") + } + bitmap := data[1:3] + indices = bitPosTwoBytes(bitmap) + if flag&byte(16) != 0 { // flagE + indices = append(indices, 16) + log.Info("Unexpected 16th child encountered in a full node") + } + data = data[3:] + } + // Reconstruct elements + elements := make([][]byte, 0, len(indices)) + for i := 0; i < len(indices); i++ { + if len(data) == 0 { + return nil, nil, errors.New("invalid data: missing size byte") + } + // Read element size + size := int(data[0]) + data = data[1:] + + // Check if we have enough data for the element + if len(data) < size { + return nil, nil, fmt.Errorf("invalid data: expected %d bytes, got %d", size, len(data)) + } + // Extract element + if size == 0 { + elements = append(elements, nil) + + // The zero-size element is practically unexpected, for node deletion + // the rlp.EmptyString is still expected. Log loudly for the potential + // programming error. + log.Error("Empty element from compressed node, please open an issue", "raw", data) + } else { + element := make([]byte, size) + copy(element, data[:size]) + data = data[size:] + elements = append(elements, element) + } + } + // Check if all data is consumed + if len(data) != 0 { + return nil, nil, errors.New("invalid data: trailing bytes") + } + return elements, indices, nil +} + +// decodeNodeFull decodes the byte stream of full value trie node. +func decodeNodeFull(data []byte) (bool, []byte, error) { + if len(data) < 1 { + return false, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag != byte(0) { + return false, nil, nil + } + return true, data[1:], nil +} + +// encodeNodeHistory encodes the history of a node. Typically, the original values +// of dirty nodes serve as the history, but this can lead to significant storage +// overhead. +// +// For full nodes, which often see only a few modified children during state +// transitions, recording the entire child set (up to 16 children at 32 bytes +// each) is inefficient. For short nodes, which often see only the value is +// modified during the state transition, recording the key part is also unnecessary. +// To compress size, we instead record the diff of the node, rather than the +// full value. It's vital to compress the overall trienode history. +// +// However, recovering a node from a series of diffs requires applying multiple +// history records, which is computationally and IO intensive. To mitigate this, we +// periodically record the full value of a node as a checkpoint. The frequency of +// these checkpoints is a tradeoff between the compression rate and read overhead. +func (s *nodeSetWithOrigin) encodeNodeHistory(root common.Hash, rate uint32) (map[common.Hash]map[string][]byte, error) { + var ( + // the set of all encoded node history elements + nodes = make(map[common.Hash]map[string][]byte) + + // encodeFullValue determines whether a node should be encoded + // in full format with a pseudo-random probabilistic algorithm. + encodeFullValue = func(owner common.Hash, path string) bool { + // For trie nodes at the first two levels of the account trie, it is very + // likely that all children are modified within a single state transition. + // In such cases, do not use diff mode. + if owner == (common.Hash{}) && len(path) < 2 { + return true + } + h := fnv.New32a() + h.Write(root.Bytes()) + h.Write(owner.Bytes()) + h.Write([]byte(path)) + return h.Sum32()%rate == 0 + } + ) + for owner, origins := range s.nodeOrigin { + var posts map[string]*trienode.Node + if owner == (common.Hash{}) { + posts = s.nodeSet.accountNodes + } else { + posts = s.nodeSet.storageNodes[owner] + } + nodes[owner] = make(map[string][]byte) + + for path, oldvalue := range origins { + n, exists := posts[path] + if !exists { + // something not expected + return nil, fmt.Errorf("node with origin is not found, %x-%v", owner, []byte(path)) + } + encodeFull := encodeFullValue(owner, path) + if !encodeFull { + // TODO(rjl493456442) the diff-mode reencoding can take non-trivial + // time, like 1-2ms per block, is there any way to mitigate the overhead? + + // Partial encoding is required, try to find the node diffs and + // fallback to the full-value encoding if fails. + // + // The partial encoding will be failed in these certain cases: + // - the node is deleted or was not-existent; + // - the node type has been changed (e.g, from short to full) + nElem, indices, diffs, err := trie.NodeDifference(oldvalue, n.Blob) + if err != nil { + encodeFull = true // fallback to the full node encoding + } else { + // Encode the node difference as the history element + addExt := nElem != 2 // fullNode + blob := encodeNodeCompressed(addExt, diffs, indices) + nodes[owner][path] = blob + } + } + if encodeFull { + // Encode the entire original value as the history element + nodes[owner][path] = encodeNodeFull(oldvalue) + } + } + } + return nodes, nil +} diff --git a/triedb/pathdb/nodes_test.go b/triedb/pathdb/nodes_test.go index 483dc4b1a6..131d0ab012 100644 --- a/triedb/pathdb/nodes_test.go +++ b/triedb/pathdb/nodes_test.go @@ -18,11 +18,13 @@ package pathdb import ( "bytes" + "math/rand" "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -126,3 +128,49 @@ func TestNodeSetWithOriginEncode(t *testing.T) { t.Fatalf("Unexpected data size, got: %d, want: %d", dec2.size, s.size) } } + +func TestEncodeFullNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i <= 16; i++ { + if rand.Intn(2) == 0 { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + } + enc := encodeNodeCompressed(true, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} + +func TestEncodeShortNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i < 2; i++ { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + enc := encodeNodeCompressed(false, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 842ac0972e..aaa64e902c 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -47,7 +47,7 @@ type nodeLoc struct { } // string returns the string representation of node location. -func (loc *nodeLoc) string() string { +func (loc nodeLoc) string() string { return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth) } @@ -200,7 +200,7 @@ func (db *Database) StateReader(root common.Hash) (database.StateReader, error) // historical state. type HistoricalStateReader struct { db *Database - reader *historyReader + reader *stateHistoryReader id uint64 } @@ -234,7 +234,7 @@ func (db *Database) HistoricReader(root common.Hash) (*HistoricalStateReader, er return &HistoricalStateReader{ id: *id, db: db, - reader: newHistoryReader(db.diskdb, db.stateFreezer), + reader: newStateHistoryReader(db.diskdb, db.stateFreezer), }, nil } @@ -318,3 +318,90 @@ func (r *HistoricalStateReader) Storage(address common.Address, key common.Hash) } return r.reader.read(newStorageIdentQuery(address, addrHash, key, keyHash), r.id, dl.stateID(), latest) } + +// HistoricalNodeReader is a wrapper over history reader, providing access to +// historical trie node data. +type HistoricalNodeReader struct { + db *Database + reader *trienodeReader + id uint64 +} + +// HistoricNodeReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricNodeReader(root common.Hash) (*HistoricalNodeReader, error) { + // Bail out if the state history hasn't been fully indexed + if db.trienodeIndexer == nil || db.trienodeFreezer == nil { + return nil, fmt.Errorf("historical state %x is not available", root) + } + if !db.trienodeIndexer.inited() { + return nil, errors.New("trienode histories haven't been fully indexed yet") + } + // - States at the current disk layer or above are directly accessible + // via `db.NodeReader`. + // + // - States older than the current disk layer (including the disk layer + // itself) are available via `db.HistoricalNodeReader`. + id := rawdb.ReadStateID(db.diskdb, root) + if id == nil { + return nil, fmt.Errorf("state %#x is not available", root) + } + // Ensure the requested trienode history is canonical, states on side chain + // are not accessible. + meta, err := readTrienodeMetadata(db.trienodeFreezer, *id+1) + if err != nil { + return nil, err // e.g., the referred trienode history has been pruned + } + if meta.parent != root { + return nil, fmt.Errorf("state %#x is not canonincal", root) + } + return &HistoricalNodeReader{ + id: *id, + db: db, + reader: newTrienodeReader(db.diskdb, db.trienodeFreezer, int(db.config.FullValueCheckpoint)), + }, nil +} + +// Node directly retrieves the trie node data associated with a particular path, +// within a particular account. An error will be returned if the read operation +// exits abnormally. Specifically, if the layer is already stale. +// +// Note: +// - the returned trie node data is not a copy, please don't modify it. +// - an error will be returned if the requested trie node is not found in database. +func (r *HistoricalNodeReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + defer func(start time.Time) { + historicalTrienodeReadTimer.UpdateSince(start) + }(time.Now()) + + // TODO(rjl493456442): Theoretically, the obtained disk layer could become stale + // within a very short time window. + // + // While reading the account data while holding `db.tree.lock` can resolve + // this issue, but it will introduce a heavy contention over the lock. + // + // Let's optimistically assume the situation is very unlikely to happen, + // and try to define a low granularity lock if the current approach doesn't + // work later. + dl := r.db.tree.bottom() + latest, h, _, err := dl.node(owner, path, 0) + if err != nil { + return nil, err + } + if h == hash { + return latest, nil + } + blob, err := r.reader.read(newTrienodeIdent(owner, string(path)), r.id, dl.stateID(), latest) + if err != nil { + return nil, err + } + // Error out if the local one is inconsistent with the target. + if crypto.Keccak256Hash(blob) != hash { + blobHex := "nil" + if len(blob) > 0 { + blobHex = hexutil.Encode(blob) + } + log.Error("Unexpected historical trie node", "owner", owner.Hex(), "path", path, "blob", blobHex) + return nil, fmt.Errorf("unexpected historical trie node: (%x %v), blob: %s", owner, path, blobHex) + } + return blob, nil +} diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index dc737c3b53..c54d8b1136 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -170,12 +170,13 @@ func (s *stateSet) accountList() []common.Hash { if list != nil { return list } - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list = s.accountListSorted; list != nil { + return list + } list = slices.SortedFunc(maps.Keys(s.accountData), common.Hash.Cmp) s.accountListSorted = list return list @@ -200,12 +201,13 @@ func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { } s.listLock.RUnlock() - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list := s.storageListSorted[accountHash]; list != nil { + return list + } list := slices.SortedFunc(maps.Keys(s.storageData[accountHash]), common.Hash.Cmp) s.storageListSorted[accountHash] = list return list @@ -340,7 +342,10 @@ func (s *stateSet) encode(w io.Writer) error { AddrHashes []common.Hash Accounts [][]byte } - var enc accounts + enc := accounts{ + AddrHashes: make([]common.Hash, 0, len(s.accountData)), + Accounts: make([][]byte, 0, len(s.accountData)), + } for addrHash, blob := range s.accountData { enc.AddrHashes = append(enc.AddrHashes, addrHash) enc.Accounts = append(enc.Accounts, blob) @@ -503,7 +508,10 @@ func (s *StateSetWithOrigin) encode(w io.Writer) error { Addresses []common.Address Accounts [][]byte } - var accounts Accounts + accounts := Accounts{ + Addresses: make([]common.Address, 0, len(s.accountOrigin)), + Accounts: make([][]byte, 0, len(s.accountOrigin)), + } for address, blob := range s.accountOrigin { accounts.Addresses = append(accounts.Addresses, address) accounts.Accounts = append(accounts.Accounts, blob) diff --git a/version/version.go b/version/version.go index dd944470ea..6c7124fca2 100644 --- a/version/version.go +++ b/version/version.go @@ -18,7 +18,7 @@ package version const ( Major = 1 // Major version component of the current release - Minor = 16 // Minor version component of the current release - Patch = 8 // Patch version component of the current release + Minor = 17 // Minor version component of the current release + Patch = 0 // Patch version component of the current release Meta = "stable" // Version metadata to append to the version string )