diff --git a/integration/benchmark/README.md b/integration/benchmark/README.md new file mode 100644 index 000000000..a1c559c2a --- /dev/null +++ b/integration/benchmark/README.md @@ -0,0 +1,79 @@ +# FSC Benchmarks + +This package contains useful micro benchmarks for the FSC runtime. + +## Benchmarking + +### Background material + +There are many useful articles about go benchmarks available online. Here just a few good starting points: +- https://gobyexample.com/testing-and-benchmarking +- https://blog.cloudflare.com/go-dont-collect-my-garbage/ +- https://mcclain.sh/posts/go-benchmarking/ + +### Run them all + +We can run all benchmarks in this package as follows: + +```bash +go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,8,16 -run=^$ ./... > plots/benchmark_gc_100.txt +``` + +#### Garbage Collection + +Some code, in particular, allocation-intensive operations may benefit from tweaking the garbage collector settings. +There is a highly recommended read about go's GC https://go.dev/doc/gc-guide. + +```bash +# Default +GOGC=100 + +# No garbage collection, use this setting only for testing! :) +GOGC=off +``` + +If we want to study the impact of different GC settings we can run the following, for example: + +```bash +GOGC=100 go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_100.txt +GOGC=off go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_100.txt +GOGC=8000 go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_8000.txt +``` + + +## Plotting + +The `plot/` directory contains a python script to visualize the benchmark results. + +### Install + +```bash +python3 -m venv env +source env/bin/activate +pip3 install -r requirements.txt +``` + +### Plot + +Run the python script and provide the input file and the output file as arguments. + +```bash +python3 plot.py benchmark_gc_off.txt benchmark_gc_off.pdf +``` + +This will generate the graph as pdf (`result_.pdf`). + +### Example + +Let's run the `ECDSASignView` benchmark as an example. +We turn garbage collection off using with `GOGC=off`, set the number of benchmark iteration with `-count=10`, and set the number of workers with `-cpu=1,2,4,8,16`. +We save the results in `benchmark_gc_off.txt`, which we use later to plot our graphs. + +Once the benchmark is finished, we use `plot/plot.py` to create the result graphs as `pdf`. + +```bash +GOGC=off go test -bench='ECDSASignView' -benchmem -count=10 -cpu=1,2,4,8,16 -run=^$ ./... > plots/benchmark_gc_off.txt +cd plots; python3 plot.py benchmark_gc_off.txt benchmark_gc_off.pdf +``` + +Happy benchmarking! diff --git a/integration/benchmark/benchmark.go b/integration/benchmark/benchmark.go new file mode 100644 index 000000000..6c1ffc075 --- /dev/null +++ b/integration/benchmark/benchmark.go @@ -0,0 +1,15 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package benchmark + +import ( + "testing" +) + +func ReportTPS(b *testing.B) { + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "TPS") +} diff --git a/integration/benchmark/grpc/grpc_bench_test.go b/integration/benchmark/grpc/grpc_bench_test.go new file mode 100644 index 000000000..7a431bde3 --- /dev/null +++ b/integration/benchmark/grpc/grpc_bench_test.go @@ -0,0 +1,153 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" + benchviews "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark/views" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/grpc" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics/disabled" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/server" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/server/protos" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" +) + +func BenchmarkGRPC(b *testing.B) { + srvEndpoint := setupServer(b) + + // we share a single connection among all client goroutines + cli, closeF := setupClient(b, srvEndpoint) + defer closeF() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + resp, err := cli.CallViewWithContext(b.Context(), "fid", nil) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + benchmark.ReportTPS(b) +} + +func setupServer(tb testing.TB) string { + tb.Helper() + + mDefaultIdentity := view.Identity("server identity") + mSigner := &benchmark.MockSigner{ + SerializeFunc: func() ([]byte, error) { + return mDefaultIdentity.Bytes(), nil + }, + SignFunc: func(bytes []byte) ([]byte, error) { + return bytes, nil + }, + } + mIdentityProvider := &benchmark.MockIdentityProvider{DefaultSigner: mDefaultIdentity} + mSigService := &benchmark.MockSignerProvider{DefaultSigner: mSigner} + + // marshaller + tm, err := server.NewResponseMarshaler(mIdentityProvider, mSigService) + require.NoError(tb, err) + require.NotNil(tb, tm) + + // setup server + grpcSrv, err := grpc.NewGRPCServer("localhost:0", grpc.ServerConfig{ + ConnectionTimeout: 0, + SecOpts: grpc.SecureOptions{ + Certificate: certPEM, + Key: keyPEM, + UseTLS: true, + }, + KaOpts: grpc.KeepaliveOptions{}, + Logger: nil, + HealthCheckEnabled: false, + }) + + require.NoError(tb, err) + require.NotNil(tb, grpcSrv) + + srv, err := server.NewViewServiceServer(tm, &server.YesPolicyChecker{}, server.NewMetrics(&disabled.Provider{}), noop.NewTracerProvider()) + require.NoError(tb, err) + require.NotNil(tb, srv) + + parms := &benchviews.CPUParams{N: 200000} + input, _ := json.Marshal(parms) + factory := &benchviews.CPUViewFactory{} + v, _ := factory.NewView(input) + + // our view manager + vm := &benchmark.MockViewManager{Constructor: func() view.View { + return v + }} + + // register view manager wit grpc impl + server.InstallViewHandler(vm, srv, noop.NewTracerProvider()) + + // register grpc impl with grpc server + protos.RegisterViewServiceServer(grpcSrv.Server(), srv) + + // start the actual grpc server + go func() { + _ = grpcSrv.Start() + }() + tb.Cleanup(grpcSrv.Stop) + + return grpcSrv.Address() +} + +func setupClient(tb testing.TB, srvEndpoint string) (*benchmark.ViewClient, func()) { + tb.Helper() + + mDefaultIdentity := view.Identity("client identity") + mSigner := &benchmark.MockSigner{ + SerializeFunc: func() ([]byte, error) { + return mDefaultIdentity.Bytes(), nil + }, + SignFunc: func(bytes []byte) ([]byte, error) { + return bytes, nil + }} + + signerIdentity, err := mSigner.Serialize() + require.NoError(tb, err) + + grpcClient, err := grpc.NewGRPCClient(grpc.ClientConfig{ + SecOpts: grpc.SecureOptions{ + ServerRootCAs: [][]byte{certPEM}, + UseTLS: true, + }, + KaOpts: grpc.KeepaliveOptions{}, + Timeout: 5 * time.Second, + AsyncConnect: false, + }) + require.NoError(tb, err) + require.NotNil(tb, grpcClient) + + conn, err := grpcClient.NewConnection(srvEndpoint) + require.NoError(tb, err) + require.NotNil(tb, conn) + + tlsCert := grpcClient.Certificate() + tlsCertHash, err := grpc.GetTLSCertHash(&tlsCert) + require.NoError(tb, err) + + cli := &benchmark.ViewClient{ + SignF: mSigner.Sign, + Creator: signerIdentity, + TLSCertHash: tlsCertHash, + Client: protos.NewViewServiceClient(conn), + } + + return cli, func() { + assert.NoError(tb, conn.Close()) + } +} diff --git a/integration/benchmark/grpc/reference_bench_test.go b/integration/benchmark/grpc/reference_bench_test.go new file mode 100644 index 000000000..0c85a8303 --- /dev/null +++ b/integration/benchmark/grpc/reference_bench_test.go @@ -0,0 +1,189 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net" + "testing" + "time" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" + benchviews "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark/views" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/server/protos" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" + "github.com/hyperledger/fabric-lib-go/common/flogging" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +var ( + keyPEM []byte + certPEM []byte +) + +func init() { + flogging.Init(flogging.Config{ + LogSpec: "grpc=error:error", + }) + + keyPEM, certPEM, _ = makeSelfSignedCert() +} + +// makeSelfSignedCert generates a localhost self-signed cert using ECDSA P-256. +// It returns the tls.Certificate and the PEM-encoded cert for the client root pool. +func makeSelfSignedCert() ([]byte, []byte, error) { + // 1. generate ECDSA private key + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + + // 2. certificate template + serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62)) + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + Organization: []string{"Local Test CA"}, + }, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + + // 3. sign it (self-signed) + derBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return nil, nil, err + } + + // 4. PEM-encode cert & private key + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + + keyBytes, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, nil, err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}) + + return keyPEM, certPEM, nil +} + +type serverImpl struct { + protos.UnimplementedViewServiceServer + + workload view.View +} + +func (s *serverImpl) ProcessCommand(ctx context.Context, command *protos.SignedCommand) (*protos.SignedCommandResponse, error) { + resp, err := s.workload.Call(nil) + if err != nil { + return nil, err + } + + // TODO include resp in signed response + _ = resp + + return &protos.SignedCommandResponse{}, nil +} + +func (s *serverImpl) StreamCommand(g grpc.BidiStreamingServer[protos.SignedCommand, protos.SignedCommandResponse]) error { + //TODO implement me + panic("implement me") +} + +func BenchmarkRef(b *testing.B) { + srvEndpoint := setupRefServer(b) + + client, closeF := setupRefClient(b, srvEndpoint) + defer closeF() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + resp, err := client.ProcessCommand(b.Context(), &protos.SignedCommand{}) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + benchmark.ReportTPS(b) +} + +func setupRefServer(tb testing.TB) string { + tb.Helper() + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(tb, err) + + serverTLS := &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + } + + // setup server + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(tb, err) + + var opts []grpc.ServerOption + opts = append(opts, grpc.Creds(credentials.NewTLS(serverTLS))) + grpcServer := grpc.NewServer(opts...) + + // create simple view + parms := &benchviews.CPUParams{N: 200000} + input, _ := json.Marshal(parms) + factory := &benchviews.CPUViewFactory{} + v, _ := factory.NewView(input) + + srv := &serverImpl{workload: v} + + protos.RegisterViewServiceServer(grpcServer, srv) + go func() { + _ = grpcServer.Serve(lis) + }() + + tb.Cleanup(func() { + grpcServer.Stop() + }) + + return lis.Addr().String() +} + +func setupRefClient(tb testing.TB, srvEndpoint string) (protos.ViewServiceClient, func()) { + tb.Helper() + + rootPool := x509.NewCertPool() + ok := rootPool.AppendCertsFromPEM(certPEM) + require.True(tb, ok) + + clientTLS := &tls.Config{ + RootCAs: rootPool, + ServerName: "localhost", // must match cert's DNSNames / SAN + } + + var clientOpts []grpc.DialOption + clientOpts = append(clientOpts, grpc.WithTransportCredentials(credentials.NewTLS(clientTLS))) + conn, err := grpc.NewClient(srvEndpoint, clientOpts...) + require.NoError(tb, err) + + return protos.NewViewServiceClient(conn), func() { + assert.NoError(tb, conn.Close()) + } +} diff --git a/integration/benchmark/grpc/view_bench_test.go b/integration/benchmark/grpc/view_bench_test.go new file mode 100644 index 000000000..bab78194e --- /dev/null +++ b/integration/benchmark/grpc/view_bench_test.go @@ -0,0 +1,31 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package test + +import ( + "testing" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" + "github.com/stretchr/testify/require" +) + +func BenchmarkView(b *testing.B) { + srvEndpoint := setupServer(b) + + // we share a single connection among all client goroutines + cli, closeF := setupClient(b, srvEndpoint) + defer closeF() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + resp, err := cli.CallViewWithContext(b.Context(), "fid", nil) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + benchmark.ReportTPS(b) +} diff --git a/integration/benchmark/grpc_view_client.go b/integration/benchmark/grpc_view_client.go new file mode 100644 index 000000000..6022221a4 --- /dev/null +++ b/integration/benchmark/grpc_view_client.go @@ -0,0 +1,91 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package benchmark + +import ( + "context" + "crypto/rand" + "fmt" + "io" + "time" + + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/proto" + protos2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/server/protos" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type ViewClient struct { + SignF func(msg []byte) ([]byte, error) + Creator []byte + TLSCertHash []byte + Client protos2.ViewServiceClient +} + +func (vc *ViewClient) CallView(fid string, input []byte) (interface{}, error) { + return vc.CallViewWithContext(context.TODO(), fid, input) +} + +func (vc *ViewClient) CallViewWithContext(ctx context.Context, fid string, input []byte) (interface{}, error) { + c := &protos2.Command{Payload: &protos2.Command_CallView{CallView: &protos2.CallView{Fid: fid, Input: input}}} + + sc, err := vc.createSignedCommand(c) + if err != nil { + return nil, err + } + + scr, err := vc.Client.ProcessCommand(ctx, sc) + if err != nil { + return nil, err + } + + commandResp := &protos2.CommandResponse{} + err = proto.Unmarshal(scr.Response, commandResp) + if err != nil { + return nil, err + } + + if commandResp.GetErr() != nil { + return nil, fmt.Errorf("error from view during process command: %s", commandResp.GetErr().GetMessage()) + } + + return commandResp.GetCallViewResponse().GetResult(), nil +} + +func (vc *ViewClient) createSignedCommand(command *protos2.Command) (*protos2.SignedCommand, error) { + nonce := make([]byte, 32) + _, err := io.ReadFull(rand.Reader, nonce) + if err != nil { + return nil, err + } + + ts := timestamppb.New(time.Now()) + if err := ts.CheckValid(); err != nil { + return nil, err + } + + command.Header = &protos2.Header{ + Timestamp: ts, + Nonce: nonce, + Creator: vc.Creator, + TlsCertHash: vc.TLSCertHash, + } + + raw, err := proto.Marshal(command) + if err != nil { + return nil, err + } + + signature, err := vc.SignF(raw) + if err != nil { + return nil, err + } + + return &protos2.SignedCommand{ + Command: raw, + Signature: signature, + }, nil +} diff --git a/integration/benchmark/mocks.go b/integration/benchmark/mocks.go new file mode 100644 index 000000000..317b5637d --- /dev/null +++ b/integration/benchmark/mocks.go @@ -0,0 +1,67 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package benchmark + +import ( + "context" + + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/sig" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" +) + +type MockSigner struct { + SerializeFunc func() ([]byte, error) + SignFunc func([]byte) ([]byte, error) +} + +func (m *MockSigner) Serialize() ([]byte, error) { + return m.SerializeFunc() +} + +func (m *MockSigner) Sign(msg []byte) ([]byte, error) { + return m.SignFunc(msg) +} + +type MockIdentityProvider struct { + DefaultSigner view.Identity +} + +func (m *MockIdentityProvider) DefaultIdentity() view.Identity { + return m.DefaultSigner +} + +func (m *MockIdentityProvider) Admins() []view.Identity { + panic("implement me") +} + +func (m *MockIdentityProvider) Clients() []view.Identity { + panic("implement me") +} + +type MockSignerProvider struct { + DefaultSigner sig.Signer +} + +func (m *MockSignerProvider) GetSigner(identity view.Identity) (sig.Signer, error) { + return m.DefaultSigner, nil +} + +type MockViewManager struct { + Constructor func() view.View +} + +func (m *MockViewManager) NewView(id string, in []byte) (view.View, error) { + return m.Constructor(), nil +} + +func (m *MockViewManager) InitiateView(view view.View, ctx context.Context) (interface{}, error) { + return view.Call(nil) +} + +func (m *MockViewManager) InitiateContext(view view.View) (view.Context, error) { + panic("implement me") +} diff --git a/integration/benchmark/node/bench_test.go b/integration/benchmark/node/bench_test.go new file mode 100644 index 000000000..7e1cf1337 --- /dev/null +++ b/integration/benchmark/node/bench_test.go @@ -0,0 +1,217 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package node + +import ( + "context" + "encoding/json" + "fmt" + "path" + "testing" + "time" + + "github.com/hyperledger-labs/fabric-smart-client/integration" + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark/views" + "github.com/hyperledger-labs/fabric-smart-client/integration/nwo/fsc" + "github.com/hyperledger-labs/fabric-smart-client/node" + viewsdk "github.com/hyperledger-labs/fabric-smart-client/platform/view/sdk/dig" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/grpc" + viewregistry "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/client" + view2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/client/cmd" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/view/grpc/server/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Benchmark(b *testing.B) { + benchmarks := []struct { + name string + factory viewregistry.Factory + params any + }{ + { + name: "noop", + factory: &views.NoopViewFactory{}, + }, + { + name: "cpu", + factory: &views.CPUViewFactory{}, + params: &views.CPUParams{N: 200000}, + }, + { + name: "sign", + factory: &views.ECDSASignViewFactory{}, + params: &views.ECDSASignParams{}, + }, + } + + testdataPath := b.TempDir() // for local debugging you can set testdataPath := "out/testdata" + nodeConfPath := path.Join(testdataPath, "fsc", "nodes", "test-node.0") + clientConfPath := path.Join(nodeConfPath, "client-config.yaml") + + // we generate our testdata + err := generateConfig(b, testdataPath) + require.NoError(b, err) + + // run all benchmarks via direct view API + for _, bm := range benchmarks { + b.Run(fmt.Sprintf("direct/%s", bm.name), func(b *testing.B) { + n, err := setupNode(b, nodeConfPath, namedFactory{ + name: bm.name, + factory: bm.factory, + }) + require.NoError(b, err) + b.Cleanup(n.Stop) + + vm, err := viewregistry.GetManager(n) + require.NoError(b, err) + + var in []byte + if bm.params != nil { + in, err = json.Marshal(bm.params) + require.NoError(b, err) + } + + b.RunParallel(func(pb *testing.PB) { + // each goroutine instantiates a dedicated view + f, err := vm.NewView(bm.name, in) + assert.NoError(b, err) + + for pb.Next() { + _, err = vm.InitiateView(f, context.Background()) + assert.NoError(b, err) + } + }) + benchmark.ReportTPS(b) + }) + + // run all benchmarks via grpc view API + b.Run(fmt.Sprintf("grpc/%s", bm.name), func(b *testing.B) { + n, err := setupNode(b, nodeConfPath, namedFactory{ + name: bm.name, + factory: bm.factory, + }) + require.NoError(b, err) + b.Cleanup(n.Stop) + + var in []byte + if bm.params != nil { + in, err = json.Marshal(bm.params) + require.NoError(b, err) + } + + // setup grpc client + cli, err := setupClient(b, clientConfPath) + require.NoError(b, err) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := cli.CallViewWithContext(b.Context(), bm.name, in) + assert.NoError(b, err) + } + }) + benchmark.ReportTPS(b) + }) + } +} + +func generateConfig(tb testing.TB, testdataDir string) error { + tb.Helper() + + fscTopology := fsc.NewTopology() + fscTopology.SetLogging("error", "") + fscTopology.AddNodeByName("test-node") + + _, err := integration.GenerateAt(8000, testdataDir, false, fscTopology) + if err != nil { + return err + } + + return nil +} + +func setupNode(tb testing.TB, confPath string, factories ...namedFactory) (*node.Node, error) { + tb.Helper() + + fsc := node.NewWithConfPath(confPath) + if err := fsc.InstallSDK(viewsdk.NewSDK(fsc)); err != nil { + return nil, err + } + + if err := fsc.Start(); err != nil { + return nil, err + } + + reg := viewregistry.GetRegistry(fsc) + + for _, f := range factories { + err := reg.RegisterFactory(f.name, f.factory) + if err != nil { + return nil, err + } + } + + return fsc, nil +} + +type namedFactory struct { + name string + factory viewregistry.Factory +} + +func setupClient(tb testing.TB, confPath string) (*benchmark.ViewClient, error) { + tb.Helper() + + config, err := view2.ConfigFromFile(confPath) + if err != nil { + return nil, err + } + + signer, err := client.NewX509SigningIdentity(config.SignerConfig.IdentityPath, config.SignerConfig.KeyPath) + if err != nil { + return nil, err + } + + signerIdentity, err := signer.Serialize() + if err != nil { + return nil, err + } + + cc := &grpc.ConnectionConfig{ + Address: config.Address, + TLSEnabled: true, + TLSRootCertFile: path.Join(config.TLSConfig.PeerCACertPath), + ConnectionTimeout: 10 * time.Second, + } + + grpcClient, err := grpc.CreateGRPCClient(cc) + if err != nil { + return nil, err + } + + conn, err := grpcClient.NewConnection(config.Address) + if err != nil { + return nil, err + } + + tlsCert := grpcClient.Certificate() + tlsCertHash, err := grpc.GetTLSCertHash(&tlsCert) + if err != nil { + return nil, err + } + + vc := &benchmark.ViewClient{ + SignF: signer.Sign, + Creator: signerIdentity, + TLSCertHash: tlsCertHash, + Client: protos.NewViewServiceClient(conn), + } + + return vc, nil +} diff --git a/integration/benchmark/plots/.gitignore b/integration/benchmark/plots/.gitignore new file mode 100644 index 000000000..1a609d2ca --- /dev/null +++ b/integration/benchmark/plots/.gitignore @@ -0,0 +1,4 @@ +env +results +*.pdf +*.txt diff --git a/integration/benchmark/plots/plot.py b/integration/benchmark/plots/plot.py new file mode 100644 index 000000000..898212664 --- /dev/null +++ b/integration/benchmark/plots/plot.py @@ -0,0 +1,82 @@ +import re +import sys +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages +from collections import defaultdict + +# ---------------------------- +# HANDLE CLI ARGUMENTS +# ---------------------------- + +if len(sys.argv) < 3: + print("Usage: python plot.py ") + sys.exit(1) + +INPUT_FILE = sys.argv[1] +OUTPUT_PDF = sys.argv[2] + +# ---------------------------- +# PARSING LOGIC +# ---------------------------- +# +# We need to extract: +# - benchmark group name (e.g. "BenchmarkSimple/parallel") +# - worker count (default = 1) +# - TPS value +# +# ---------------------------- + +# dictionary: +# group_name → { worker → [tps runs] } +groups = defaultdict(lambda: defaultdict(list)) + +# Matches all forms: +# BenchmarkSimple/parallel-4 ... 140240 TPS +# BenchmarkParallelWork-12 ... 13246 TPS +pattern = re.compile(r"(Benchmark[^\s/-]+(?:/[^\s/-]+)?)" # group name + r"(?:-(\d+))?" # optional worker/cpu suffix + r".*?([\d.]+)\s+TPS", re.IGNORECASE) + +with open(INPUT_FILE, "r") as f: + for line in f: + m = pattern.search(line) + if not m: + continue + + group = m.group(1) + worker_str = m.group(2) + tps = float(m.group(3)) + + worker = int(worker_str) if worker_str else 1 + + groups[group][worker].append(tps) + +with PdfPages(OUTPUT_PDF) as pdf: + + for group_name, worker_dict in sorted(groups.items()): + + # Sort and compute stats + worker_counts = sorted(worker_dict.keys()) + tps_means = [np.mean(worker_dict[c]) for c in worker_counts] + tps_stddev = [np.std(worker_dict[c]) for c in worker_counts] + + print(f"\nGroup: {group_name}") + print(" Worker counts:", worker_counts) + print(" TPS means:", tps_means) + + # ------------- + # Plot 1: TPS + # ------------- + + plt.figure(figsize=(10, 6)) + plt.errorbar(worker_counts, tps_means, yerr=tps_stddev, fmt="o-", capsize=6) + plt.xlabel("Worker Count") + plt.ylabel("Average TPS") + plt.title(f"{group_name}") + plt.grid(True) + plt.tight_layout() + pdf.savefig() + plt.close() + +print(f"\nSaved multi-benchmark PDF to: {OUTPUT_PDF}") \ No newline at end of file diff --git a/integration/benchmark/plots/requirements.txt b/integration/benchmark/plots/requirements.txt new file mode 100644 index 000000000..84f35b4af --- /dev/null +++ b/integration/benchmark/plots/requirements.txt @@ -0,0 +1,12 @@ +contourpy==1.3.3 +cycler==0.12.1 +fonttools==4.61.1 +kiwisolver==1.4.9 +matplotlib==3.10.8 +np==1.0.2 +numpy==2.3.5 +packaging==25.0 +pillow==12.0.0 +pyparsing==3.2.5 +python-dateutil==2.9.0.post0 +six==1.17.0 diff --git a/integration/benchmark/views/cpu_bench.go b/integration/benchmark/views/cpu_bench.go new file mode 100644 index 000000000..02be582f1 --- /dev/null +++ b/integration/benchmark/views/cpu_bench.go @@ -0,0 +1,48 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "encoding/json" + + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" +) + +type CPUParams struct { + N int +} + +type CPUView struct { + params CPUParams +} + +func (q *CPUView) Call(viewCtx view.Context) (interface{}, error) { + k := doWork(q.params.N) + _ = k + return "OK", nil +} + +// doWork just burns CPU cycles +func doWork(n int) uint64 { + var x uint64 + for i := 0; i < n; i++ { + x += uint64(i * i) + } + return x +} + +type CPUViewFactory struct{} + +func (c *CPUViewFactory) NewView(in []byte) (view.View, error) { + + f := &CPUView{} + if err := json.Unmarshal(in, &f.params); err != nil { + return nil, err + } + + return f, nil +} diff --git a/integration/benchmark/views/cpu_bench_test.go b/integration/benchmark/views/cpu_bench_test.go new file mode 100644 index 000000000..ff6572fa9 --- /dev/null +++ b/integration/benchmark/views/cpu_bench_test.go @@ -0,0 +1,30 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "encoding/json" + "testing" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" +) + +func BenchmarkCPU(b *testing.B) { + + f := &CPUViewFactory{} + // tune up/down for longer/shorter ops + p := &CPUParams{N: 200000} + input, _ := json.Marshal(p) + + b.RunParallel(func(pb *testing.PB) { + v, _ := f.NewView(input) + for pb.Next() { + _, _ = v.Call(nil) + } + }) + benchmark.ReportTPS(b) +} diff --git a/integration/benchmark/views/noop_bench.go b/integration/benchmark/views/noop_bench.go new file mode 100644 index 000000000..6a2f6a0c2 --- /dev/null +++ b/integration/benchmark/views/noop_bench.go @@ -0,0 +1,24 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" +) + +type NoopView struct { +} + +func (q *NoopView) Call(viewCtx view.Context) (interface{}, error) { + return "OK", nil +} + +type NoopViewFactory struct{} + +func (c *NoopViewFactory) NewView(_ []byte) (view.View, error) { + return &NoopView{}, nil +} diff --git a/integration/benchmark/views/noop_bench_test.go b/integration/benchmark/views/noop_bench_test.go new file mode 100644 index 000000000..d14d6bc8e --- /dev/null +++ b/integration/benchmark/views/noop_bench_test.go @@ -0,0 +1,24 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "testing" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" +) + +func BenchmarkNoop(b *testing.B) { + f := &NoopViewFactory{} + b.RunParallel(func(pb *testing.PB) { + v, _ := f.NewView(nil) + for pb.Next() { + _, _ = v.Call(nil) + } + }) + benchmark.ReportTPS(b) +} diff --git a/integration/benchmark/views/signer_bench.go b/integration/benchmark/views/signer_bench.go new file mode 100644 index 000000000..d6766ec6b --- /dev/null +++ b/integration/benchmark/views/signer_bench.go @@ -0,0 +1,63 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "io" + + "github.com/hyperledger-labs/fabric-smart-client/platform/view/view" +) + +type ECDSASignParams struct { +} + +type ECDSASignView struct { + params ECDSASignParams + + pr *ecdsa.PrivateKey + r io.Reader +} + +var msgBytes = []byte("hello, world") + +func (q *ECDSASignView) Call(viewCtx view.Context) (interface{}, error) { + hash := sha256.Sum256(msgBytes) + + sig, err := ecdsa.SignASN1(q.r, q.pr, hash[:]) + if err != nil { + return "error", err + } + + return base64.StdEncoding.EncodeToString(sig), nil +} + +type ECDSASignViewFactory struct{} + +func (c *ECDSASignViewFactory) NewView(in []byte) (view.View, error) { + + f := &ECDSASignView{} + if err := json.Unmarshal(in, &f.params); err != nil { + return nil, err + } + + // setup signing key + f.r = rand.Reader + pk, err := ecdsa.GenerateKey(elliptic.P256(), f.r) + if err != nil { + return nil, err + } + + f.pr = pk + + return f, nil +} diff --git a/integration/benchmark/views/signer_bench_test.go b/integration/benchmark/views/signer_bench_test.go new file mode 100644 index 000000000..9435a2bbd --- /dev/null +++ b/integration/benchmark/views/signer_bench_test.go @@ -0,0 +1,29 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package views + +import ( + "encoding/json" + "testing" + + "github.com/hyperledger-labs/fabric-smart-client/integration/benchmark" +) + +func BenchmarkECDSASign(b *testing.B) { + f := &ECDSASignViewFactory{} + p := &ECDSASignParams{} + input, _ := json.Marshal(p) + + b.RunParallel(func(pb *testing.PB) { + // note that each benchmark goroutine gets their own + v, _ := f.NewView(input) + for pb.Next() { + _, _ = v.Call(nil) + } + }) + benchmark.ReportTPS(b) +}