Merge pull request #71 from datachainlab/faster-get-boundary-height

Optimize search for hard fork boundary

Author: izquierdo-datachain

module/fork_spec.go

@@ -3,11 +3,15 @@ package module
 import (
 	"context"
 	"fmt"
-	"github.com/cockroachdb/errors"
-	"github.com/hyperledger-labs/yui-relayer/log"
+	"math"
+	"math/big"
 	"os"
 	"slices"
 	"strconv"
+
+	"github.com/cockroachdb/errors"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/hyperledger-labs/yui-relayer/log"
 )
 
 type Network string
@@ -225,6 +229,7 @@ func FindTargetForkSpec(forkSpecs []*ForkSpec, height uint64, timestamp uint64)
 var boundaryHeightCache = make(map[uint64]uint64)
 
 func GetBoundaryHeight(ctx context.Context, headerFn getHeaderFn, currentHeight uint64, currentForkSpec ForkSpec) (*BoundaryHeight, error) {
+	var err error
 	logger := log.GetLogger()
 	boundaryHeight := uint64(0)
 	if condition, ok := currentForkSpec.GetHeightOrTimestamp().(*ForkSpec_Height); ok {
@@ -235,27 +240,139 @@ func GetBoundaryHeight(ctx context.Context, headerFn getHeaderFn, currentHeight
 			boundaryHeight = v
 		} else {
 			logger.DebugContext(ctx, "seek fork height", "currentHeight", currentHeight, "ts", ts)
-			for i := int64(currentHeight); i >= 0; i-- {
-				h, err := headerFn(ctx, uint64(i))
-				if err != nil {
-					return nil, err
-				}
-				if MilliTimestamp(h) == ts {
-					boundaryHeight = h.Number.Uint64()
-					logger.DebugContext(ctx, "seek fork height found", "currentHeight", currentHeight, "ts", ts, "boundaryHeight", boundaryHeight)
-					boundaryHeightCache[ts] = boundaryHeight
-					break
-				} else if MilliTimestamp(h) < ts {
-					boundaryHeight = h.Number.Uint64() + 1
-					logger.DebugContext(ctx, "seek fork height found", "currentHeight", currentHeight, "ts", ts, "boundaryHeight", boundaryHeight)
-					boundaryHeightCache[ts] = boundaryHeight
-					break
-				}
+			boundaryHeight, err = searchBoundaryHeight(ctx, currentHeight, ts, headerFn)
+			if err != nil {
+				return nil, err
 			}
+			boundaryHeightCache[ts] = boundaryHeight
 		}
 	}
 	return &BoundaryHeight{
 		Height:          boundaryHeight,
 		CurrentForkSpec: currentForkSpec,
 	}, nil
 }
+
+func searchBoundaryHeight(ctx context.Context, currentHeight uint64, targetTs uint64, headerFn getHeaderFn) (uint64, error) {
+	// There are potentially many blocks between the boundary and the current
+	// blocks. Also, finding the timestamp for a particular block is expensive
+	// as it requires an RPC call to a node.
+	//
+	// Thus, this implementation aims to prune a large number of blocks from the
+	// search space by estimating the distance between the boundary and the
+	// current block (based on the average rate of block production) and jumping
+	// directly to a candidate block at that distance. In case of a miss, all
+	// blocks on one side of the candidate can be discarded, and a new attempt
+	// can be made by re-estimating the new distance and jumping to a candidate
+	// on the other side.
+	//
+	// Theoretical worst-case performance is O(N), but since the rate of block
+	// production can be predicted with high accuracy, this implementation is
+	// expected to be faster than binary search in practice.
+	var (
+		position       uint64        = currentHeight     // candidate block number currently under consideration
+		low            uint64        = 0                 // inclusive lower bound of the current search range
+		high           uint64        = currentHeight + 1 // exclusive upper bound of the current search range
+		previousHeader *types.Header                     // header of the block seen in the previous iteration
+	)
+
+	// Loop invariant:
+	//
+	//     0 <= low <= position < high <= currentHeight + 1
+	//     &&
+	//     low <= result < high
+	//
+	// Bound function (decreases in each iteration, and is always >= 0):
+	//
+	//      high - low
+	for low < high {
+		currentHeader, err := headerFn(ctx, uint64(position))
+		if err != nil {
+			return 0, err
+		}
+
+		currentTs := MilliTimestamp(currentHeader)
+		if currentTs == targetTs {
+			return currentHeader.Number.Uint64(), nil
+		}
+
+		distance := estimateDistance(previousHeader, currentHeader, targetTs)
+
+		if currentTs > targetTs {
+			// Jump to a lower block.
+			high = position
+
+			// Since these are unsigned, position-distance might underflow.
+			if low+distance > position {
+				position = low
+			} else {
+				position = position - distance
+			}
+		} else {
+			// Jump to a higher block.
+			low = position + 1
+
+			position = position + distance
+
+			if position >= high {
+				position = high - 1
+			}
+		}
+
+		previousHeader = currentHeader
+	}
+
+	// If no block with an exact timestamp match was found, then we want the
+	// earliest block that's _after_ the target timestamp.
+	return low, nil
+}
+
+// estimateDistance returns the estimated number of blocks between the block indicated by currentHeader
+// and the boundary block nearest to targetTs. It assumes that previousHeader either is nil, or refers to
+// a different block than currentHeader.
+func estimateDistance(previousHeader, currentHeader *types.Header, targetTs uint64) uint64 {
+	if previousHeader == nil {
+		return 1
+	}
+
+	var (
+		timeDiffPrevCur   uint64 // milliseconds between the previous and current blocks
+		timeDiffTargetCur uint64 // milliseconds between the current block and target timestamp
+	)
+
+	currentTs := MilliTimestamp(currentHeader)
+	previousTs := MilliTimestamp(previousHeader)
+
+	blockCountPrevCurBig := new(big.Int).Sub(previousHeader.Number, currentHeader.Number)
+	blockCountPrevCurBig = blockCountPrevCurBig.Abs(blockCountPrevCurBig)
+	blockCountPrevCur, _ := blockCountPrevCurBig.Float64()
+
+	if currentTs > previousTs {
+		timeDiffPrevCur = currentTs - previousTs
+	} else {
+		timeDiffPrevCur = previousTs - currentTs
+	}
+
+	if timeDiffPrevCur == 0 {
+		// Found two different blocks with the same timestamp. The distance
+		// should be at least 1 to avoid getting stuck in the current block.
+		return 1
+	}
+
+	if currentTs > targetTs {
+		timeDiffTargetCur = currentTs - targetTs
+	} else {
+		timeDiffTargetCur = targetTs - currentTs
+	}
+
+	avgBlocksPerMs := blockCountPrevCur / float64(timeDiffPrevCur)
+
+	if avgBlocksPerMs > 0 {
+		return uint64(math.Ceil(avgBlocksPerMs * float64(timeDiffTargetCur)))
+	}
+
+	// Blocks are being produced so slowly that the current block is still expected
+	// to be the latest block at any future timestamp. Return 1 to avoid getting stuck
+	// in the current block.
+	return 1
+}

module/fork_spec_test.go

@@ -3,11 +3,15 @@ package module
 import (
 	"context"
 	"fmt"
+	"math"
+	"math/big"
+	"testing"
+
+	"errors"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/hyperledger-labs/yui-relayer/log"
 	"github.com/stretchr/testify/suite"
-	"math/big"
-	"testing"
 )
 
 type ForkSpecTestSuite struct {
@@ -475,3 +479,75 @@ func (ts *ForkSpecTestSuite) Test_Success_GetBoundaryEpochs_After_Maxwell_2() {
 	ts.Require().Equal(epochs.PreviousEpochBlockNumber(2000), uint64(1000))
 	ts.Require().Equal(epochs.PreviousEpochBlockNumber(3000), uint64(1000))
 }
+
+func (ts *ForkSpecTestSuite) Test_searchBoundaryHeight_Success() {
+	ctx := context.Background()
+	var currentHeight uint64 = 200
+	var targetTs uint64 = 123_456
+	headerFn := func(ctx context.Context, height uint64) (*types.Header, error) {
+		return &types.Header{Number: big.NewInt(int64(height)), Time: height}, nil
+	}
+	height, err := searchBoundaryHeight(ctx, currentHeight, targetTs, headerFn)
+	ts.NoError(err)
+	ts.Equal(uint64(124), height)
+}
+
+func (ts *ForkSpecTestSuite) Test_searchBoundaryHeight_Error() {
+	ctx := context.Background()
+	var currentHeight uint64 = 200
+	var targetTs uint64 = 123_456
+	headerFn := func(ctx context.Context, height uint64) (*types.Header, error) {
+		return nil, errors.New("test error")
+	}
+	_, err := searchBoundaryHeight(ctx, currentHeight, targetTs, headerFn)
+	ts.Error(err)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_NoPrevious() {
+	currentHeader := &types.Header{Number: big.NewInt(int64(123)), Time: 123_456}
+	targetTs := 123_456
+	distance := estimateDistance(nil, currentHeader, uint64(targetTs))
+	ts.Require().Equal(uint64(1), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_MoveLowAfterLow() {
+	previousHeader := &types.Header{Number: big.NewInt(int64(200)), Time: 60_000}
+	currentHeader := &types.Header{Number: big.NewInt(int64(100)), Time: 20_000}
+	distance := estimateDistance(previousHeader, currentHeader, 10_987_000)
+	ts.Require().Equal(uint64(23), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_MoveHighAfterLow() {
+	previousHeader := &types.Header{Number: big.NewInt(int64(200)), Time: 60_000}
+	currentHeader := &types.Header{Number: big.NewInt(int64(100)), Time: 20_000}
+	distance := estimateDistance(previousHeader, currentHeader, 20_720_000)
+	ts.Require().Equal(uint64(2), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_MoveHighAfterHigh() {
+	previousHeader := &types.Header{Number: big.NewInt(int64(100)), Time: 85_000}
+	currentHeader := &types.Header{Number: big.NewInt(int64(200)), Time: 90_000}
+	distance := estimateDistance(previousHeader, currentHeader, 97_123_000)
+	ts.Require().Equal(uint64(143), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_MoveLowAfterHigh() {
+	previousHeader := &types.Header{Number: big.NewInt(int64(100)), Time: 85_000}
+	currentHeader := &types.Header{Number: big.NewInt(int64(200)), Time: 90_000}
+	distance := estimateDistance(previousHeader, currentHeader, 87_123_000)
+	ts.Require().Equal(uint64(58), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_SameTimestamp() {
+	previousHeader := &types.Header{Number: big.NewInt(int64(100)), Time: 99_900}
+	currentHeader := &types.Header{Number: big.NewInt(int64(200)), Time: 99_900}
+	distance := estimateDistance(previousHeader, currentHeader, 50_000)
+	ts.Require().Equal(uint64(1), distance)
+}
+
+func (ts *ForkSpecTestSuite) Test_estimateDistance_MinimumRate() {
+	previousHeader := &types.Header{Number: big.NewInt(17), Time: math.MaxUint64}
+	currentHeader := &types.Header{Number: big.NewInt(16), Time: 0}
+	distance := estimateDistance(previousHeader, currentHeader, 50_000)
+	ts.Require().Equal(uint64(1), distance)
+}