diff --git a/Makefile b/Makefile index f621527aca..479a92c54d 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ all: build build-replay-env test-gen-proofs @touch .make/all .PHONY: build -build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daprovider daserver autonomous-auctioneer bidder-client datool el-proxy mockexternalsigner seq-coordinator-invalidate nitro-val seq-coordinator-manager dbconv genesis-generator) +build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daprovider daserver autonomous-auctioneer bidder-client datool blobtool el-proxy mockexternalsigner seq-coordinator-invalidate nitro-val seq-coordinator-manager dbconv genesis-generator) @printf $(done) .PHONY: build-node-deps @@ -334,6 +334,9 @@ $(output_root)/bin/el-proxy: $(DEP_PREDICATE) build-node-deps $(output_root)/bin/datool: $(DEP_PREDICATE) build-node-deps go build $(GOLANG_PARAMS) -o $@ "$(CURDIR)/cmd/datool" +$(output_root)/bin/blobtool: $(DEP_PREDICATE) build-node-deps + go build $(GOLANG_PARAMS) -o $@ "$(CURDIR)/cmd/blobtool" + $(output_root)/bin/genesis-generator: $(DEP_PREDICATE) build-node-deps go build $(GOLANG_PARAMS) -o $@ "$(CURDIR)/cmd/genesis-generator" diff --git a/cmd/blobtool/blobtool.go b/cmd/blobtool/blobtool.go new file mode 100644 index 0000000000..94da9eb8a4 --- /dev/null +++ b/cmd/blobtool/blobtool.go @@ -0,0 +1,234 @@ +// Copyright 2025, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE.md + +// This is a command line tool for testing beacon/blobs and blob_sidecars endpoints. +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + flag "github.com/spf13/pflag" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/kzg4844" + + "github.com/offchainlabs/nitro/cmd/util/confighelpers" + "github.com/offchainlabs/nitro/util/blobs" + "github.com/offchainlabs/nitro/util/headerreader" +) + +func main() { + args := os.Args + if len(args) < 2 { + fmt.Println("Usage: blobtool [fetch] ...") + os.Exit(1) + } + + var err error + switch strings.ToLower(args[1]) { + case "fetch": + err = fetchBlobs(args[2:]) + default: + err = fmt.Errorf("unknown command '%s', valid commands are: fetch", args[1]) + } + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +type FetchConfig struct { + BeaconURL string `koanf:"beacon-url"` + Slot uint64 `koanf:"slot"` + VersionedHashes []string `koanf:"versioned-hashes"` + UseLegacyEndpoint bool `koanf:"use-legacy-endpoint"` + CompareEndpoints bool `koanf:"compare-endpoints"` +} + +func parseFetchConfig(args []string) (*FetchConfig, error) { + f := flag.NewFlagSet("blobtool fetch", flag.ContinueOnError) + f.String("beacon-url", "", "Beacon Chain RPC URL. For example with --beacon-url=http://localhost, an RPC call will be made to http://localhost/eth/v1/beacon/blobs") + f.Uint64("slot", 0, "Beacon chain slot number to fetch blobs from") + f.StringSlice("versioned-hashes", []string{}, "Comma-separated list of versioned hashes to fetch (optional - fetches all if not provided)") + f.Bool("use-legacy-endpoint", false, "Use the legacy blob_sidecars endpoint") + f.Bool("compare-endpoints", false, "Fetch using both endpoints and compare results") + + k, err := confighelpers.BeginCommonParse(f, args) + if err != nil { + return nil, err + } + + var config FetchConfig + if err := confighelpers.EndCommonParse(k, &config); err != nil { + return nil, err + } + + if config.BeaconURL == "" { + return nil, fmt.Errorf("--beacon-url is required") + } + if config.Slot == 0 { + return nil, fmt.Errorf("--slot is required") + } + + return &config, nil +} + +func fetchBlobs(args []string) error { + config, err := parseFetchConfig(args) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + versionedHashes := make([]common.Hash, len(config.VersionedHashes)) + for i, hashStr := range config.VersionedHashes { + if !common.IsHexAddress(hashStr) && len(hashStr) != 66 { + return fmt.Errorf("invalid versioned hash at index %d: %s", i, hashStr) + } + versionedHashes[i] = common.HexToHash(hashStr) + } + + if config.UseLegacyEndpoint && len(versionedHashes) == 0 { + return fmt.Errorf("--versioned-hashes is required when using --use-legacy-endpoint") + } + + if config.CompareEndpoints { + if len(versionedHashes) == 0 { + return fmt.Errorf("--versioned-hashes is required when using --compare-endpoints") + } + return compareEndpoints(ctx, config, versionedHashes) + } + + blobClientConfig := headerreader.BlobClientConfig{ + BeaconUrl: config.BeaconURL, + UseLegacyEndpoint: config.UseLegacyEndpoint, + } + + blobClient, err := headerreader.NewBlobClient(blobClientConfig, nil) + if err != nil { + return fmt.Errorf("failed to create blob client: %w", err) + } + + if err := blobClient.Initialize(ctx); err != nil { + return fmt.Errorf("failed to initialize blob client: %w", err) + } + + endpointType := "new blobs" + if config.UseLegacyEndpoint { + endpointType = "legacy blob_sidecars" + } + + if len(versionedHashes) > 0 { + fmt.Printf("Fetching %d blobs for slot %d using %s endpoint...\n", len(versionedHashes), config.Slot, endpointType) + } else { + fmt.Printf("Fetching all blobs for slot %d using %s endpoint...\n", config.Slot, endpointType) + } + + startTime := time.Now() + fetchedBlobs, err := blobClient.GetBlobsBySlot(ctx, config.Slot, versionedHashes) + if err != nil { + return fmt.Errorf("failed to fetch blobs: %w", err) + } + duration := time.Since(startTime) + + fmt.Printf("Successfully fetched %d blobs in %v\n", len(fetchedBlobs), duration) + + for i, blob := range fetchedBlobs { + _, hashes, err := blobs.ComputeCommitmentsAndHashes([]kzg4844.Blob{blob}) + if err != nil { + return fmt.Errorf("failed to compute commitment for blob %d: %w", i, err) + } + if len(versionedHashes) > 0 { + fmt.Printf("Blob %d: versioned_hash=%s (computed=%s), size=%d bytes\n", i, versionedHashes[i].Hex(), hashes[0].Hex(), len(blob)) + } else { + fmt.Printf("Blob %d: versioned_hash=%s, size=%d bytes\n", i, hashes[0].Hex(), len(blob)) + } + } + + return nil +} + +func compareEndpoints(ctx context.Context, config *FetchConfig, versionedHashes []common.Hash) error { + fmt.Println("Comparing legacy blob_sidecars and new blobs endpoints...") + fmt.Println() + + legacyConfig := headerreader.BlobClientConfig{ + BeaconUrl: config.BeaconURL, + UseLegacyEndpoint: true, + } + legacyClient, err := headerreader.NewBlobClient(legacyConfig, nil) + if err != nil { + return fmt.Errorf("failed to create legacy blob client: %w", err) + } + if err := legacyClient.Initialize(ctx); err != nil { + return fmt.Errorf("failed to initialize legacy blob client: %w", err) + } + + fmt.Println("Fetching with legacy blob_sidecars endpoint...") + legacyStart := time.Now() + legacyBlobs, err := legacyClient.GetBlobsBySlot(ctx, config.Slot, versionedHashes) + legacyDuration := time.Since(legacyStart) + if err != nil { + return fmt.Errorf("failed to fetch blobs with legacy endpoint: %w", err) + } + fmt.Printf("✓ Legacy endpoint: fetched %d blobs in %v\n", len(legacyBlobs), legacyDuration) + fmt.Println() + + newConfig := headerreader.BlobClientConfig{ + BeaconUrl: config.BeaconURL, + UseLegacyEndpoint: false, + } + newClient, err := headerreader.NewBlobClient(newConfig, nil) + if err != nil { + return fmt.Errorf("failed to create new blob client: %w", err) + } + if err := newClient.Initialize(ctx); err != nil { + return fmt.Errorf("failed to initialize new blob client: %w", err) + } + + fmt.Println("Fetching with new blobs endpoint...") + newStart := time.Now() + newBlobs, err := newClient.GetBlobsBySlot(ctx, config.Slot, versionedHashes) + newDuration := time.Since(newStart) + if err != nil { + return fmt.Errorf("failed to fetch blobs with new endpoint: %w", err) + } + fmt.Printf("✓ New endpoint: fetched %d blobs in %v\n", len(newBlobs), newDuration) + fmt.Println() + + if len(legacyBlobs) != len(newBlobs) { + return fmt.Errorf("blob count mismatch: legacy=%d, new=%d", len(legacyBlobs), len(newBlobs)) + } + + fmt.Println("Comparing blob data...") + for i := range legacyBlobs { + if legacyBlobs[i] != newBlobs[i] { + return fmt.Errorf("blob %d data mismatch", i) + } + _, hashes, err := blobs.ComputeCommitmentsAndHashes([]kzg4844.Blob{legacyBlobs[i]}) + if err != nil { + return fmt.Errorf("failed to compute hash for blob %d: %w", i, err) + } + fmt.Printf(" Blob %d: ✓ identical (%s)\n", i, hashes[0].Hex()) + } + + fmt.Println() + fmt.Printf("Performance comparison:\n") + fmt.Printf(" Legacy endpoint: %v\n", legacyDuration) + fmt.Printf(" New endpoint: %v\n", newDuration) + if newDuration < legacyDuration { + improvement := float64(legacyDuration-newDuration) / float64(legacyDuration) * 100 + fmt.Printf(" New endpoint is %.1f%% faster\n", improvement) + } else { + slower := float64(newDuration-legacyDuration) / float64(legacyDuration) * 100 + fmt.Printf(" New endpoint is %.1f%% slower\n", slower) + } + + return nil +} diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index b8881e2ec9..eb0d2d7c4b 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -35,6 +35,7 @@ type BlobClient struct { secondaryBeaconUrl *url.URL httpClient atomic.Pointer[http.Client] authorization string + useLegacyEndpoint bool // Filled in in Initialize() genesisTime uint64 @@ -49,6 +50,7 @@ type BlobClientConfig struct { SecondaryBeaconUrl string `koanf:"secondary-beacon-url"` BlobDirectory string `koanf:"blob-directory"` Authorization string `koanf:"authorization"` + UseLegacyEndpoint bool `koanf:"use-legacy-endpoint"` } var DefaultBlobClientConfig = BlobClientConfig{ @@ -56,6 +58,7 @@ var DefaultBlobClientConfig = BlobClientConfig{ SecondaryBeaconUrl: "", BlobDirectory: "", Authorization: "", + UseLegacyEndpoint: false, } func BlobClientAddOptions(prefix string, f *pflag.FlagSet) { @@ -63,6 +66,7 @@ func BlobClientAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".secondary-beacon-url", DefaultBlobClientConfig.SecondaryBeaconUrl, "Backup beacon Chain RPC URL to use for fetching blobs (normally on port 3500) when unable to fetch from primary") f.String(prefix+".blob-directory", DefaultBlobClientConfig.BlobDirectory, "Full path of the directory to save fetched blobs") f.String(prefix+".authorization", DefaultBlobClientConfig.Authorization, "Value to send with the HTTP Authorization: header for Beacon REST requests, must include both scheme and scheme parameters") + f.Bool(prefix+".use-legacy-endpoint", DefaultBlobClientConfig.UseLegacyEndpoint, "Use the legacy blob_sidecars endpoint instead of the blobs endpoint") } func NewBlobClient(config BlobClientConfig, ec *ethclient.Client) (*BlobClient, error) { @@ -92,6 +96,7 @@ func NewBlobClient(config BlobClientConfig, ec *ethclient.Client) (*BlobClient, beaconUrl: beaconUrl, secondaryBeaconUrl: secondaryBeaconUrl, authorization: config.Authorization, + useLegacyEndpoint: config.UseLegacyEndpoint, blobDirectory: config.BlobDirectory, } blobClient.httpClient.Store(&http.Client{}) @@ -102,14 +107,15 @@ type fullResult[T any] struct { Data T `json:"data"` } -func beaconRequest[T interface{}](b *BlobClient, ctx context.Context, beaconPath string) (T, error) { - // Unfortunately, methods on a struct can't be generic. - +func beaconRequest[T interface{}](b *BlobClient, ctx context.Context, beaconPath string, queryParams url.Values) (T, error) { var empty T - fetchData := func(url url.URL) (*http.Response, error) { - url.Path = path.Join(url.Path, beaconPath) - req, err := http.NewRequestWithContext(ctx, "GET", url.String(), http.NoBody) + fetchData := func(beaconUrl url.URL) (*http.Response, error) { + beaconUrl.Path = path.Join(beaconUrl.Path, beaconPath) + if queryParams != nil { + beaconUrl.RawQuery = queryParams.Encode() + } + req, err := http.NewRequestWithContext(ctx, "GET", beaconUrl.String(), http.NoBody) if err != nil { return nil, err } @@ -170,7 +176,23 @@ func (b *BlobClient) GetBlobs(ctx context.Context, blockHash common.Hash, versio return nil, errors.New("BlobClient hasn't been initialized") } slot := (header.Time - b.genesisTime) / b.secondsPerSlot - blobs, err := b.blobSidecars(ctx, slot, versionedHashes) + + return b.GetBlobsBySlot(ctx, slot, versionedHashes) +} + +// Get blobs for a specific beacon chain slot. +func (b *BlobClient) GetBlobsBySlot(ctx context.Context, slot uint64, versionedHashes []common.Hash) ([]kzg4844.Blob, error) { + if b.secondsPerSlot == 0 { + return nil, errors.New("BlobClient hasn't been initialized") + } + + var blobs []kzg4844.Blob + var err error + if b.useLegacyEndpoint { + blobs, err = b.blobSidecars(ctx, slot, versionedHashes) + } else { + blobs, err = b.getBlobs(ctx, slot, versionedHashes) + } if err != nil { // Creates a new http client to avoid reusing the same transport layer connection in the next request. // This strategy can be useful if there is a network load balancer in front of the beacon chain server. @@ -178,11 +200,54 @@ func (b *BlobClient) GetBlobs(ctx context.Context, blockHash common.Hash, versio // we can potentially connect to a different, and healthy, beacon chain node in the next request. b.httpClient.Store(&http.Client{}) - return nil, fmt.Errorf("error fetching blobs in %d l1 block: %w", header.Number, err) + return nil, fmt.Errorf("error fetching blobs for slot %d: %w", slot, err) } return blobs, nil } +func (b *BlobClient) getBlobs(ctx context.Context, slot uint64, versionedHashes []common.Hash) ([]kzg4844.Blob, error) { + queryParams := url.Values{} + for _, hash := range versionedHashes { + queryParams.Add("versioned_hashes", hash.Hex()) + } + + response, err := beaconRequest[[]hexutil.Bytes](b, ctx, fmt.Sprintf("/eth/v1/beacon/blobs/%d", slot), queryParams) + if err != nil { + // #nosec G115 + roughAgeOfSlot := uint64(time.Now().Unix()) - (b.genesisTime + slot*b.secondsPerSlot) + if roughAgeOfSlot > b.secondsPerSlot*32*4096 { + return nil, fmt.Errorf("beacon client in getBlobs got error fetching older blobs in slot: %d, an archive endpoint is required, please refer to https://docs.arbitrum.io/run-arbitrum-node/l1-ethereum-beacon-chain-rpc-providers, err: %w", slot, err) + } else { + return nil, fmt.Errorf("beacon client in getBlobs got error fetching non-expired blobs in slot: %d, err: %w", slot, err) + } + } + + if len(versionedHashes) > 0 && len(response) != len(versionedHashes) { + return nil, fmt.Errorf("expected %d blobs for slot %d but got %d", len(versionedHashes), slot, len(response)) + } + + output := make([]kzg4844.Blob, len(response)) + for i, blobData := range response { + if len(blobData) != len(output[i]) { + return nil, fmt.Errorf("blob at index %d has incorrect length %d, expected %d", i, len(blobData), len(output[i])) + } + copy(output[i][:], blobData) + + if len(versionedHashes) > 0 { + commitment, err := kzg4844.BlobToCommitment(&output[i]) + if err != nil { + return nil, fmt.Errorf("failed to compute commitment for blob %d: %w", i, err) + } + computedHash := blobs.CommitmentToVersionedHash(commitment) + if computedHash != versionedHashes[i] { + return nil, fmt.Errorf("blob %d versioned hash mismatch: expected %s, got %s", i, versionedHashes[i].Hex(), computedHash.Hex()) + } + } + } + + return output, nil +} + type blobResponseItem struct { BlockRoot string `json:"block_root"` Index jsonapi.Uint64String `json:"index"` @@ -197,7 +262,7 @@ type blobResponseItem struct { const trailingCharsOfResponse = 25 func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHashes []common.Hash) ([]kzg4844.Blob, error) { - rawData, err := beaconRequest[json.RawMessage](b, ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot)) + rawData, err := beaconRequest[json.RawMessage](b, ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot), nil) if err != nil || len(rawData) == 0 { // blobs are pruned after 4096 epochs (1 epoch = 32 slots), we determine if the requested slot was to be pruned by a non-archive endpoint // #nosec G115 @@ -306,13 +371,13 @@ type getSpecResponse struct { } func (b *BlobClient) Initialize(ctx context.Context) error { - genesis, err := beaconRequest[genesisResponse](b, ctx, "/eth/v1/beacon/genesis") + genesis, err := beaconRequest[genesisResponse](b, ctx, "/eth/v1/beacon/genesis", nil) if err != nil { return fmt.Errorf("error calling beacon client to get genesisTime: %w", err) } b.genesisTime = uint64(genesis.GenesisTime) - spec, err := beaconRequest[getSpecResponse](b, ctx, "/eth/v1/config/spec") + spec, err := beaconRequest[getSpecResponse](b, ctx, "/eth/v1/config/spec", nil) if err != nil { return fmt.Errorf("error calling beacon client to get secondsPerSlot: %w", err) }