diff --git a/.gitignore b/.gitignore index f7e4cb674..8a5f694b5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .vscode .build *.iml - +.venv # ignoring all generated artifacts in integration tests integration/**/out/ diff --git a/integration/benchmark/README.md b/integration/benchmark/README.md index a1c559c2a..98f029b85 100644 --- a/integration/benchmark/README.md +++ b/integration/benchmark/README.md @@ -36,7 +36,7 @@ If we want to study the impact of different GC settings we can run the following ```bash GOGC=100 go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_100.txt -GOGC=off go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_100.txt +GOGC=off go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_off.txt GOGC=8000 go test -bench=. -benchmem -count=10 -timeout=20m -cpu=1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,48,64 -run=^$ ./... > plots/benchmark_gc_8000.txt ``` diff --git a/integration/benchmark/grpc/grpc_bench_test.go b/integration/benchmark/grpc/grpc_bench_test.go index 7a431bde3..116070c9e 100644 --- a/integration/benchmark/grpc/grpc_bench_test.go +++ b/integration/benchmark/grpc/grpc_bench_test.go @@ -23,8 +23,9 @@ import ( "go.opentelemetry.io/otel/trace/noop" ) -func BenchmarkGRPC(b *testing.B) { - srvEndpoint := setupServer(b) +func BenchmarkGRPCSingleConnectionCPU(b *testing.B) { + srvEndpoint := setupServer(b, "cpu") + b.ResetTimer() // we share a single connection among all client goroutines cli, closeF := setupClient(b, srvEndpoint) @@ -40,7 +41,60 @@ func BenchmarkGRPC(b *testing.B) { benchmark.ReportTPS(b) } -func setupServer(tb testing.TB) string { +func BenchmarkGRPCMultiConnectionCPU(b *testing.B) { + srvEndpoint := setupServer(b, "cpu") + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + // each goroutine gets its own client + connection + cli, closeF := setupClient(b, srvEndpoint) + defer closeF() + + for pb.Next() { + resp, err := cli.CallViewWithContext(b.Context(), "fid", nil) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + + benchmark.ReportTPS(b) +} + +// --- ECDSA Workload Benchmarks --- + +func BenchmarkGRPCSingleConnectionECDSA(b *testing.B) { + srvEndpoint := setupServer(b, "ecdsa") + b.ResetTimer() + + cli, closeF := setupClient(b, srvEndpoint) + defer closeF() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + resp, err := cli.CallViewWithContext(b.Context(), "fid", nil) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + benchmark.ReportTPS(b) +} + +func BenchmarkGRPCMultiConnectionECDSA(b *testing.B) { + srvEndpoint := setupServer(b, "ecdsa") + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + cli, closeF := setupClient(b, srvEndpoint) + defer closeF() + + for pb.Next() { + resp, err := cli.CallViewWithContext(b.Context(), "fid", nil) + require.NoError(b, err) + require.NotNil(b, resp) + } + }) + benchmark.ReportTPS(b) +} + +func setupServer(tb testing.TB, workloadType string) string { tb.Helper() mDefaultIdentity := view.Identity("server identity") @@ -80,10 +134,21 @@ func setupServer(tb testing.TB) string { require.NoError(tb, err) require.NotNil(tb, srv) - parms := &benchviews.CPUParams{N: 200000} - input, _ := json.Marshal(parms) - factory := &benchviews.CPUViewFactory{} - v, _ := factory.NewView(input) + var v view.View + switch workloadType { + case "cpu": + parms := &benchviews.CPUParams{N: 200000} + input, _ := json.Marshal(parms) + factory := &benchviews.CPUViewFactory{} + v, _ = factory.NewView(input) + case "ecdsa": + parms := &benchviews.ECDSASignParams{} + input, _ := json.Marshal(parms) + factory := &benchviews.ECDSASignViewFactory{} + v, _ = factory.NewView(input) + default: + tb.Fatalf("unknown workload type: %s", workloadType) + } // our view manager vm := &benchmark.MockViewManager{Constructor: func() view.View { diff --git a/integration/benchmark/grpc/view_bench_test.go b/integration/benchmark/grpc/view_bench_test.go index bab78194e..66d07f5d0 100644 --- a/integration/benchmark/grpc/view_bench_test.go +++ b/integration/benchmark/grpc/view_bench_test.go @@ -14,7 +14,7 @@ import ( ) func BenchmarkView(b *testing.B) { - srvEndpoint := setupServer(b) + srvEndpoint := setupServer(b, "cpu") // we share a single connection among all client goroutines cli, closeF := setupClient(b, srvEndpoint) diff --git a/integration/benchmark/node/bench_test.go b/integration/benchmark/node/bench_test.go index 7e1cf1337..6740d8607 100644 --- a/integration/benchmark/node/bench_test.go +++ b/integration/benchmark/node/bench_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/require" ) -func Benchmark(b *testing.B) { +func BenchmarkNode(b *testing.B) { benchmarks := []struct { name string factory viewregistry.Factory @@ -92,7 +92,7 @@ func Benchmark(b *testing.B) { }) // run all benchmarks via grpc view API - b.Run(fmt.Sprintf("grpc/%s", bm.name), func(b *testing.B) { + b.Run(fmt.Sprintf("grpcSingleConnection/%s", bm.name), func(b *testing.B) { n, err := setupNode(b, nodeConfPath, namedFactory{ name: bm.name, factory: bm.factory, @@ -107,8 +107,9 @@ func Benchmark(b *testing.B) { } // setup grpc client - cli, err := setupClient(b, clientConfPath) + cli, closeF, err := setupClient(b, clientConfPath) require.NoError(b, err) + b.Cleanup(closeF) b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -118,6 +119,34 @@ func Benchmark(b *testing.B) { }) benchmark.ReportTPS(b) }) + + b.Run(fmt.Sprintf("grpcMultiConnection/%s", bm.name), func(b *testing.B) { + n, err := setupNode(b, nodeConfPath, namedFactory{ + name: bm.name, + factory: bm.factory, + }) + require.NoError(b, err) + b.Cleanup(n.Stop) + + var in []byte + if bm.params != nil { + in, err = json.Marshal(bm.params) + require.NoError(b, err) + } + + b.RunParallel(func(pb *testing.PB) { + // setup grpc client + cli, closeF, err := setupClient(b, clientConfPath) + require.NoError(b, err) + b.Cleanup(closeF) + + for pb.Next() { + _, err := cli.CallViewWithContext(b.Context(), bm.name, in) + assert.NoError(b, err) + } + }) + benchmark.ReportTPS(b) + }) } } @@ -165,22 +194,22 @@ type namedFactory struct { factory viewregistry.Factory } -func setupClient(tb testing.TB, confPath string) (*benchmark.ViewClient, error) { +func setupClient(tb testing.TB, confPath string) (*benchmark.ViewClient, func(), error) { tb.Helper() config, err := view2.ConfigFromFile(confPath) if err != nil { - return nil, err + return nil, nil, err } signer, err := client.NewX509SigningIdentity(config.SignerConfig.IdentityPath, config.SignerConfig.KeyPath) if err != nil { - return nil, err + return nil, nil, err } signerIdentity, err := signer.Serialize() if err != nil { - return nil, err + return nil, nil, err } cc := &grpc.ConnectionConfig{ @@ -192,18 +221,18 @@ func setupClient(tb testing.TB, confPath string) (*benchmark.ViewClient, error) grpcClient, err := grpc.CreateGRPCClient(cc) if err != nil { - return nil, err + return nil, nil, err } conn, err := grpcClient.NewConnection(config.Address) if err != nil { - return nil, err + return nil, nil, err } tlsCert := grpcClient.Certificate() tlsCertHash, err := grpc.GetTLSCertHash(&tlsCert) if err != nil { - return nil, err + return nil, nil, err } vc := &benchmark.ViewClient{ @@ -212,6 +241,11 @@ func setupClient(tb testing.TB, confPath string) (*benchmark.ViewClient, error) TLSCertHash: tlsCertHash, Client: protos.NewViewServiceClient(conn), } + closeFunc := func() { + if err := conn.Close(); err != nil { + fmt.Printf("failed to close connection: %v\n", err) + } + } - return vc, nil + return vc, closeFunc, nil } diff --git a/integration/benchmark/plots/plot.py b/integration/benchmark/plots/plot.py index 898212664..cccdfc0c0 100644 --- a/integration/benchmark/plots/plot.py +++ b/integration/benchmark/plots/plot.py @@ -16,27 +16,42 @@ INPUT_FILE = sys.argv[1] OUTPUT_PDF = sys.argv[2] +# ---------------------------- +# EXTRACT GOGC FROM FILENAME +# ---------------------------- +# +# Expected filenames: +# benchmark_gc_100.txt +# benchmark_gc_off.txt +# +# ---------------------------- + +gc_match = re.search(r"benchmark_gc_([^./]+)", INPUT_FILE) +if gc_match: + GOGC_LABEL = f"GOGC={gc_match.group(1)}" +else: + GOGC_LABEL = "GOGC=unknown" + # ---------------------------- # PARSING LOGIC # ---------------------------- # -# We need to extract: -# - benchmark group name (e.g. "BenchmarkSimple/parallel") +# We extract: +# - benchmark group name # - worker count (default = 1) -# - TPS value +# - TPS values # # ---------------------------- -# dictionary: # group_name → { worker → [tps runs] } groups = defaultdict(lambda: defaultdict(list)) -# Matches all forms: -# BenchmarkSimple/parallel-4 ... 140240 TPS -# BenchmarkParallelWork-12 ... 13246 TPS -pattern = re.compile(r"(Benchmark[^\s/-]+(?:/[^\s/-]+)?)" # group name - r"(?:-(\d+))?" # optional worker/cpu suffix - r".*?([\d.]+)\s+TPS", re.IGNORECASE) +pattern = re.compile( + r"(Benchmark(?:[^\s/-]+)?(?:/[^\s/-]+)*)" # group name + r"(?:-(\d+))?" # optional worker suffix + r".*?([\d.]+)\s+TPS", + re.IGNORECASE, +) with open(INPUT_FILE, "r") as f: for line in f: @@ -49,14 +64,15 @@ tps = float(m.group(3)) worker = int(worker_str) if worker_str else 1 - groups[group][worker].append(tps) -with PdfPages(OUTPUT_PDF) as pdf: +# ---------------------------- +# PLOTTING +# ---------------------------- +with PdfPages(OUTPUT_PDF) as pdf: for group_name, worker_dict in sorted(groups.items()): - # Sort and compute stats worker_counts = sorted(worker_dict.keys()) tps_means = [np.mean(worker_dict[c]) for c in worker_counts] tps_stddev = [np.std(worker_dict[c]) for c in worker_counts] @@ -65,18 +81,26 @@ print(" Worker counts:", worker_counts) print(" TPS means:", tps_means) - # ------------- - # Plot 1: TPS - # ------------- - plt.figure(figsize=(10, 6)) - plt.errorbar(worker_counts, tps_means, yerr=tps_stddev, fmt="o-", capsize=6) + plt.errorbar( + worker_counts, + tps_means, + yerr=tps_stddev, + fmt="o-", + capsize=6, + ) + plt.xlabel("Worker Count") plt.ylabel("Average TPS") - plt.title(f"{group_name}") + + # Benchmark name + GC config in header + plt.title(group_name) + plt.suptitle(GOGC_LABEL, fontsize=12, fontweight="bold", y=0.98) + plt.grid(True) - plt.tight_layout() + plt.tight_layout(rect=[0, 0, 1, 0.95]) + pdf.savefig() plt.close() -print(f"\nSaved multi-benchmark PDF to: {OUTPUT_PDF}") \ No newline at end of file +print(f"\nSaved multi-benchmark PDF to: {OUTPUT_PDF}") diff --git a/integration/benchmark/plots/plot_all.sh b/integration/benchmark/plots/plot_all.sh new file mode 100755 index 000000000..5aa58e074 --- /dev/null +++ b/integration/benchmark/plots/plot_all.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +PLOT_SCRIPT="plot.py" + +for txt in benchmark_gc_*.txt; do + # Strip .txt extension + base="${txt%.txt}" + pdf="${base}.pdf" + + echo "Generating ${pdf} from ${txt}" + + python3 "${PLOT_SCRIPT}" "${txt}" "${pdf}" +done + +echo "All plots generated ✔"