loadgen: Allow shuffling signal benchmark order (#704)

* loadgen: Allow shuffling signal benchmark order

Adds a new `-shuffle` flag to randomize the order of signals in the
benchmark runs. This is useful for cases where multiple instances of the
loadgen are run concurrently, allowing more varied benchmarking.

Signed-off-by: Marc Lopez Rubio <[email protected]>

* Add PR

Signed-off-by: Marc Lopez Rubio <[email protected]>

---------

Signed-off-by: Marc Lopez Rubio <[email protected]>

Author: marclop

loadgen/cmd/otelbench/.chloggen/f_add-shuffle-flag.yaml

@@ -0,0 +1,22 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add `-shuffle` flag to randomize the order of benchmarks"
+
+# It is mandatory to specify the component. Do not change this.
+component: otelbench
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [704]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Adds a new `-shuffle` flag to randomize the order of benchmarks. This allows concurrent
+  runs to potentially target different signals, allowing multiple signals to be benchmarked
+  at once. For non-concurrent benchmarks, there shouldn't be a significant difference due
+  the execution order.

loadgen/cmd/otelbench/Makefile

@@ -1,6 +1,6 @@
 include ../../../Makefile.Common
 
-VERSION := v0.2.2
+VERSION := v0.3.0
 
 .PHONY: get-version
 get-version:
@@ -17,4 +17,4 @@ chlog-validate: $(CHLOGGEN) # Validate the new changelog entry
 
 .PHONY: chlog-preview
 chlog-preview: $(CHLOGGEN) # Preview the changelog entry in CHANGELOG.md
-	$(CHLOGGEN) update --dry
+	$(CHLOGGEN) update --dry

loadgen/cmd/otelbench/flags.go

@@ -48,6 +48,7 @@ var Config struct {
 	ExporterOTLPHTTP bool
 
 	ConcurrencyList []int
+	Shuffle         bool
 
 	Telemetry TelemetryConfig
 }
@@ -132,6 +133,8 @@ func Init() error {
 	flag.BoolVar(&Config.Metrics, "metrics", true, "benchmark metrics")
 	flag.BoolVar(&Config.Traces, "traces", true, "benchmark traces")
 
+	flag.BoolVar(&Config.Shuffle, "shuffle", false, "shuffle the order of benchmarks. This is useful for concurrent runs.")
+
 	// `concurrency` is similar to `agents` config in apmbench
 	// Each value passed into `concurrency` list will be used as loadgenreceiver `concurrency` config
 	flag.Func("concurrency", "comma-separated `list` of concurrency (number of simulated agents) to run each benchmark with. Supports numeric values (e.g., \"1,4,8\"), \"auto\" to use available CPU cores, or \"auto:Nx\" for multipliers (e.g., \"auto:2x\" for double, \"auto:0.5x\" for half)",

loadgen/cmd/otelbench/main.go

@@ -22,6 +22,7 @@ import (
 	_ "embed"
 	"flag"
 	"fmt"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"os"
@@ -200,9 +201,17 @@ func main() {
 
 	// Get the benchmark count from -test.count flag
 	count := getBenchCount()
+
+	signals := getSignals()
+	exporters := getExporters()
+	if Config.Shuffle {
+		rand.Shuffle(len(signals), func(i, j int) {
+			signals[i], signals[j] = signals[j], signals[i]
+		})
+	}
 	for _, concurrency := range Config.ConcurrencyList {
-		for _, signal := range getSignals() {
-			for _, exporter := range getExporters() {
+		for _, signal := range signals {
+			for _, exporter := range exporters {
 				benchName := fullBenchmarkName(signal, exporter, concurrency)
 				for i := 0; i < count; i++ {
 					t := time.Now().UTC()