Skip to content

Implement the RPCBlockHeaderSubscriber for indexing finalized results #728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 80 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
80 commits
Select commit Hold shift + click to select a range
a276412
Implement the RPCBlockHeaderSubscriber for indexing finalized results
m-Peter Jan 21, 2025
7d36012
Rename RPCBlockHeaderSubscriber to RPCBlockTrackingSubscriber
m-Peter Jan 23, 2025
75a1d18
Remove redundant fetching of block header prior to SubscribeBlockHead…
m-Peter Jan 23, 2025
ce83ebb
Improve error handling in RPCBlockTrackingSubscriber subscription
m-Peter Jan 23, 2025
d159467
Simplify RPCBlockTrackingSubscriber by embedding RPCEventSubscriber f…
m-Peter Jan 23, 2025
5e8f388
Optimize RPCBlockTrackingSubscriber to avoid fetch EVM tx events for …
m-Peter Jan 24, 2025
bfe6188
Improve length checks for EVM related events
m-Peter Jan 27, 2025
037c234
Close events channel in RPCBlockTrackingSubscriber
m-Peter Jan 27, 2025
6afde4d
Improve error handling to use the gRPC status code instead of strings
m-Peter Jan 27, 2025
ff927f7
Simplify constructor of RPCBlockTrackingSubscriber
m-Peter Jan 27, 2025
4555263
Remove redundant fields from RPCBlockTrackingSubscriber
m-Peter Jan 27, 2025
34dbde5
Make us of GetEventsForBlockIDs method instead of GetEventsForHeightR…
m-Peter Jan 27, 2025
bf4626c
Add description on RPCBlockTrackingSubscriber type
m-Peter Jan 27, 2025
1f084be
Refactor function for fetching EVM events on a given block header
m-Peter Jan 27, 2025
a8329e4
Make PoC configurable
peterargue Jan 28, 2025
fa39603
fix enabled conditional
peterargue Jan 28, 2025
d68f2ba
Merge pull request #737 from onflow/petera/making-poc-configurable
peterargue Jan 28, 2025
823e965
improve error reconnection handling
peterargue Jan 29, 2025
91b1ec0
Remove redundant start-testnet & start-mainnet make recipes
m-Peter Jan 30, 2025
2ddd052
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Jan 30, 2025
97f5f93
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Jan 30, 2025
1e977a9
retry getting events on NotFound and ResourceExhausted
peterargue Feb 3, 2025
f978871
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Feb 13, 2025
23b4c87
add system to verify soft finality events were eventually sealed
peterargue Feb 15, 2025
d98d8c5
handle case when sealed stream is ahead
peterargue Feb 18, 2025
2f12447
make verification configurable
peterargue Feb 21, 2025
a4766eb
review feedback
peterargue Feb 21, 2025
44aa626
Merge pull request #757 from onflow/petera/add-sealed-data-verification
peterargue Feb 21, 2025
6711c92
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Feb 21, 2025
219d720
handle verifier reconnects after AN reboot
peterargue Feb 21, 2025
0e90079
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Mar 5, 2025
fc4ead5
fix whitespace issue from resolving conflicts
peterargue Mar 5, 2025
5d95d07
Check system tx if block is missing EVM block events
peterargue Mar 5, 2025
4b59bcb
update from review feedback
peterargue Mar 5, 2025
d736faf
tidy
peterargue Mar 5, 2025
70674b9
tidy tests
peterargue Mar 5, 2025
7318ee3
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Mar 7, 2025
f4b1c5f
Merge branch 'mpeter/poc-index-finalized-block-results' into peter/po…
peterargue Mar 7, 2025
ace558d
Updates for new keystore
peterargue Mar 7, 2025
d728f14
add block height to error
peterargue Mar 7, 2025
62f6916
updates to handle system tx failure correctly
peterargue Mar 7, 2025
a4a807f
updates for verifying these blocks
peterargue Mar 7, 2025
e10fbf7
add logging
peterargue Mar 7, 2025
8e04341
make verifier wait for reexecuting data when force-start-height is used
peterargue Mar 7, 2025
1b20a6b
use correct unsealed start height
peterargue Mar 7, 2025
7226a6b
also check sealed heights
peterargue Mar 7, 2025
05b38f2
update to fixed version of sdk
peterargue Mar 8, 2025
7ea340a
fix tidy check
peterargue Mar 8, 2025
fd55071
reduce logging
peterargue Mar 8, 2025
1457b3e
re-add verifier start block
peterargue Mar 12, 2025
0071e40
update to sdk v1.4.0
peterargue Mar 12, 2025
5f04f5e
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Mar 13, 2025
08373b3
Merge branch 'mpeter/poc-index-finalized-block-results' into peter/po…
peterargue Mar 13, 2025
4bd108e
Merge pull request #771 from onflow/peter/poc-handle-failing-system-tx
peterargue Mar 13, 2025
6826497
Merge branch 'main' into mpeter/poc-index-finalized-block-results
peterargue Mar 13, 2025
1e2a0fd
fix crash when starting ahead of access node
peterargue Mar 13, 2025
f3a5b7f
Update eth_syncing to return false for currentBlock >= highestBlock
peterargue Apr 17, 2025
3ec09b9
Fix panic in debug_traceBlockByNumber when tracer is not set
peterargue Apr 22, 2025
7b1c6f1
Add flag to disable enforcing the minimum gas price
peterargue May 5, 2025
b8c6187
add docs for flag to readme
peterargue May 5, 2025
f8a3eb1
add EnforceGasPrice in testing configs
peterargue May 5, 2025
3d6f09b
Merge pull request #811 from onflow/peter/add-flag-enforce-gas-price-…
peterargue May 6, 2025
760893a
Bump onflow/flow-go & onflow/go-ethereum dependency to v1.15.9
m-Peter Apr 23, 2025
4741e96
Fix breaking changes due to Pectra update
m-Peter Apr 23, 2025
c33b839
Construct debug tracer with proper chain config based on EVM height
m-Peter Apr 24, 2025
b93e13d
Construct replayer tracer with proper chain config based on EVM height
m-Peter Apr 24, 2025
82106ab
Use EVM block timestamps to detect if we are post Pectra
m-Peter Apr 25, 2025
8ad6517
Remove IsPrague helper in favor of timestamp-based Pectra upgrage
m-Peter Apr 28, 2025
e6a368e
Bump onflow/go-ethereum dependency to v1.15.10
m-Peter May 2, 2025
c00b0f7
Merge branch 'main' into mpeter/poc-index-finalized-block-results
m-Peter May 7, 2025
237faaa
Bump flow-go version for Pectra upgrade
m-Peter May 7, 2025
c95ea08
Improve readability of resolveBlockNumber helper function
m-Peter May 8, 2025
b7e6803
Bump flow-go-sdk version to v1.4.0
m-Peter May 8, 2025
f7dc819
Merge branch 'mpeter/poc-index-finalized-block-results' into feature/…
m-Peter May 8, 2025
2fa4139
Merge pull request #816 from onflow/feature/pectra-upgrade-soft-final…
j1010001 May 8, 2025
3400afc
Create ValidationOptions for each tx submission
m-Peter May 8, 2025
f39360d
Merge pull request #819 from onflow/mpeter/fix-pectra-rules-tx-valida…
j1010001 May 8, 2025
a2914c1
Return authorization list for SetCodeTx type
m-Peter May 9, 2025
4398ef5
Add comments to describe the tx type checks in NewTransaction
m-Peter May 12, 2025
092753c
Merge pull request #822 from onflow/mpeter/display-set-code-tx-auth-l…
j1010001 May 12, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,8 @@ e2e-test:

.PHONY: check-tidy
check-tidy:
go mod tidy
git diff --exit-code
cd tests
go mod tidy
go mod tidy -v
cd tests; go mod tidy -v
git diff --exit-code

.PHONY: build
Expand Down
97 changes: 39 additions & 58 deletions api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,15 +295,12 @@ func (b *BlockChainAPI) GetTransactionByBlockNumberAndIndex(
return nil, err
}

if blockNumber < rpc.EarliestBlockNumber {
latestBlockNumber, err := b.blocks.LatestEVMHeight()
if err != nil {
return handleError[*ethTypes.Transaction](err, l, b.collector)
}
blockNumber = rpc.BlockNumber(latestBlockNumber)
height, err := resolveBlockNumber(blockNumber, b.blocks)
if err != nil {
return handleError[*ethTypes.Transaction](err, l, b.collector)
}

block, err := b.blocks.GetByHeight(uint64(blockNumber))
block, err := b.blocks.GetByHeight(height)
if err != nil {
return handleError[*ethTypes.Transaction](err, l, b.collector)
}
Expand Down Expand Up @@ -387,6 +384,7 @@ func (b *BlockChainAPI) GetBlockByHash(
// - When blockNr is -2 the chain latest block is returned.
// - When blockNr is -3 the chain finalized block is returned.
// - When blockNr is -4 the chain safe block is returned.
// - When blockNr is -5 the chain earliest block is returned.
// - When fullTx is true all transactions in the block are returned, otherwise
// only the transaction hash is returned.
func (b *BlockChainAPI) GetBlockByNumber(
Expand All @@ -403,13 +401,9 @@ func (b *BlockChainAPI) GetBlockByNumber(
return nil, err
}

height := uint64(blockNumber)
var err error
if blockNumber < 0 {
height, err = b.blocks.LatestEVMHeight()
if err != nil {
return handleError[*ethTypes.Block](err, l, b.collector)
}
height, err := resolveBlockNumber(blockNumber, b.blocks)
if err != nil {
return handleError[*ethTypes.Block](err, l, b.collector)
}

block, err := b.blocks.GetByHeight(height)
Expand All @@ -430,7 +424,7 @@ func (b *BlockChainAPI) GetBlockByNumber(
func (b *BlockChainAPI) GetBlockReceipts(
ctx context.Context,
blockNumberOrHash rpc.BlockNumberOrHash,
) ([]map[string]interface{}, error) {
) ([]map[string]any, error) {
l := b.logger.With().
Str("endpoint", "getBlockReceipts").
Str("hash", blockNumberOrHash.String()).
Expand All @@ -442,29 +436,29 @@ func (b *BlockChainAPI) GetBlockReceipts(

height, err := resolveBlockTag(&blockNumberOrHash, b.blocks, b.logger)
if err != nil {
return handleError[[]map[string]interface{}](err, l, b.collector)
return handleError[[]map[string]any](err, l, b.collector)
}

block, err := b.blocks.GetByHeight(height)
if err != nil {
return handleError[[]map[string]interface{}](err, l, b.collector)
return handleError[[]map[string]any](err, l, b.collector)
}

receipts := make([]map[string]interface{}, len(block.TransactionHashes))
receipts := make([]map[string]any, len(block.TransactionHashes))
for i, hash := range block.TransactionHashes {
tx, err := b.transactions.Get(hash)
if err != nil {
return handleError[[]map[string]interface{}](err, l, b.collector)
return handleError[[]map[string]any](err, l, b.collector)
}

receipt, err := b.receipts.GetByTransactionID(hash)
if err != nil {
return handleError[[]map[string]interface{}](err, l, b.collector)
return handleError[[]map[string]any](err, l, b.collector)
}

receipts[i], err = ethTypes.MarshalReceipt(receipt, tx)
if err != nil {
return handleError[[]map[string]interface{}](err, l, b.collector)
return handleError[[]map[string]any](err, l, b.collector)
}
}

Expand Down Expand Up @@ -510,15 +504,12 @@ func (b *BlockChainAPI) GetBlockTransactionCountByNumber(
return nil, err
}

if blockNumber < rpc.EarliestBlockNumber {
latestBlockNumber, err := b.blocks.LatestEVMHeight()
if err != nil {
return handleError[*hexutil.Uint](err, l, b.collector)
}
blockNumber = rpc.BlockNumber(latestBlockNumber)
height, err := resolveBlockNumber(blockNumber, b.blocks)
if err != nil {
return handleError[*hexutil.Uint](err, l, b.collector)
}

block, err := b.blocks.GetByHeight(uint64(blockNumber))
block, err := b.blocks.GetByHeight(height)
if err != nil {
return handleError[*hexutil.Uint](err, l, b.collector)
}
Expand Down Expand Up @@ -562,18 +553,13 @@ func (b *BlockChainAPI) Call(
return handleError[hexutil.Bytes](err, l, b.collector)
}

tx, err := encodeTxFromArgs(args)
if err != nil {
return handleError[hexutil.Bytes](err, l, b.collector)
}

// Default address in case user does not provide one
from := b.config.Coinbase
if args.From != nil {
from = *args.From
}

res, err := b.evm.Call(tx, from, height, stateOverrides, blockOverrides)
res, err := b.evm.Call(args, from, height, stateOverrides, blockOverrides)
if err != nil {
return handleError[hexutil.Bytes](err, l, b.collector)
}
Expand Down Expand Up @@ -643,10 +629,15 @@ func (b *BlockChainAPI) GetLogs(
latest := big.NewInt(int64(h))

// if special value, use latest block number
if from.Cmp(models.EarliestBlockNumber) < 0 {
if from.Cmp(models.EarliestBlockNumber) == 0 {
from = big.NewInt(0)
} else if from.Cmp(models.PendingBlockNumber) < 0 {
from = latest
}
if to.Cmp(models.EarliestBlockNumber) < 0 {

if to.Cmp(models.EarliestBlockNumber) == 0 {
to = big.NewInt(0)
} else if to.Cmp(models.PendingBlockNumber) < 0 {
to = latest
}
Comment on lines 631 to 642
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Earliest handled, but Finalized and Safe silently map to latest

The new branch keeps the previous rule “anything < Pending ⇒ latest”, then adds a special‐case for Earliest.
Consequences:

  • -3 (finalized) and -4 (safe) are now treated exactly the same as -2 (latest).
  • That deviates from the semantics exposed by other endpoints (see comment above GetBlockByNumber).

If the intention is to support finalized/safe later, the current behaviour may mis-lead API users.

-} else if from.Cmp(models.PendingBlockNumber) < 0 {
-    from = latest
+} else if from.Cmp(models.FinalizedBlockNumber) == 0 {
+    // map to the most recent finalized height
+    from = latest  // TODO: replace once finalized height is tracked
+} else if from.Cmp(models.SafeBlockNumber) == 0 {
+    // map to the most recent safe height
+    from = latest  // TODO: replace once safe height is tracked
+} else if from.Cmp(models.PendingBlockNumber) < 0 {
+    from = latest
 }

Introduce dedicated branches (even if the value ultimately falls back to
latest) to make the intent explicit and future refactors safer.

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// if special value, use latest block number
if from.Cmp(models.EarliestBlockNumber) < 0 {
if from.Cmp(models.EarliestBlockNumber) == 0 {
from = big.NewInt(0)
} else if from.Cmp(models.PendingBlockNumber) < 0 {
from = latest
}
if to.Cmp(models.EarliestBlockNumber) < 0 {
if to.Cmp(models.EarliestBlockNumber) == 0 {
to = big.NewInt(0)
} else if to.Cmp(models.PendingBlockNumber) < 0 {
to = latest
}
// if special value, use latest block number
if from.Cmp(models.EarliestBlockNumber) == 0 {
from = big.NewInt(0)
} else if from.Cmp(models.FinalizedBlockNumber) == 0 {
// map to the most recent finalized height
from = latest // TODO: replace once finalized height is tracked
} else if from.Cmp(models.SafeBlockNumber) == 0 {
// map to the most recent safe height
from = latest // TODO: replace once safe height is tracked
} else if from.Cmp(models.PendingBlockNumber) < 0 {
from = latest
}
if to.Cmp(models.EarliestBlockNumber) == 0 {
to = big.NewInt(0)
} else if to.Cmp(models.PendingBlockNumber) < 0 {
to = latest
}


Expand Down Expand Up @@ -722,11 +713,6 @@ func (b *BlockChainAPI) EstimateGas(
return handleError[hexutil.Uint64](err, l, b.collector)
}

tx, err := encodeTxFromArgs(args)
if err != nil {
return hexutil.Uint64(BlockGasLimit), nil // return block gas limit
}

// Default address in case user does not provide one
from := b.config.Coinbase
if args.From != nil {
Expand All @@ -742,7 +728,7 @@ func (b *BlockChainAPI) EstimateGas(
return handleError[hexutil.Uint64](err, l, b.collector)
}

estimatedGas, err := b.evm.EstimateGas(tx, from, height, stateOverrides)
estimatedGas, err := b.evm.EstimateGas(args, from, height, stateOverrides)
if err != nil {
return handleError[hexutil.Uint64](err, l, b.collector)
}
Expand Down Expand Up @@ -807,14 +793,9 @@ func (b *BlockChainAPI) FeeHistory(
)
}

lastBlockNumber := uint64(lastBlock)
var err error
if lastBlock < 0 {
// From the special block tags, we only support "latest".
lastBlockNumber, err = b.blocks.LatestEVMHeight()
if err != nil {
return handleError[*ethTypes.FeeHistoryResult](err, l, b.collector)
}
lastBlockNumber, err := resolveBlockNumber(lastBlock, b.blocks)
if err != nil {
return handleError[*ethTypes.FeeHistoryResult](err, l, b.collector)
}

var (
Expand Down Expand Up @@ -965,7 +946,7 @@ func (b *BlockChainAPI) prepareBlockResponse(
Nonce: types.BlockNonce{0x1},
Timestamp: hexutil.Uint64(block.Timestamp),
BaseFeePerGas: hexutil.Big(*models.BaseFeePerGas),
LogsBloom: types.LogsBloom([]*types.Log{}),
LogsBloom: types.CreateBloom(&types.Receipt{}).Bytes(),
Miner: evmTypes.CoinbaseAddress.ToCommon(),
Sha3Uncles: types.EmptyUncleHash,
}
Expand All @@ -983,19 +964,19 @@ func (b *BlockChainAPI) prepareBlockResponse(

if len(transactions) > 0 {
totalGasUsed := hexutil.Uint64(0)
logs := make([]*types.Log, 0)
receipts := types.Receipts{}
for _, tx := range transactions {
txReceipt, err := b.receipts.GetByTransactionID(tx.Hash)
if err != nil {
return nil, err
}
totalGasUsed += hexutil.Uint64(txReceipt.GasUsed)
logs = append(logs, txReceipt.Logs...)
receipts = append(receipts, txReceipt.ToGethReceipt())
blockSize += tx.Size()
}
blockResponse.GasUsed = totalGasUsed
// TODO(m-Peter): Consider if its worthwhile to move this in storage.
blockResponse.LogsBloom = types.LogsBloom(logs)
blockResponse.LogsBloom = types.MergeBloom(receipts).Bytes()
}
blockResponse.Size = hexutil.Uint64(rlp.ListSize(blockSize))

Expand Down Expand Up @@ -1056,17 +1037,17 @@ func (b *BlockChainAPI) GetUncleByBlockHashAndIndex(
ctx context.Context,
blockHash common.Hash,
index hexutil.Uint,
) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
) (map[string]any, error) {
return map[string]any{}, nil
}

// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index.
func (b *BlockChainAPI) GetUncleByBlockNumberAndIndex(
ctx context.Context,
blockNumber rpc.BlockNumber,
index hexutil.Uint,
) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
) (map[string]any, error) {
return map[string]any{}, nil
}

// MaxPriorityFeePerGas returns a suggestion for a gas tip cap for dynamic fee transactions.
Expand Down
Loading