Skip to content

Commit f85c979

Browse files
0xMALVEE0xMALVEE
committed
Merge branch 'master' into iceberg-icewarp
2 parents 5c86796 + d2a3b49 commit f85c979

4 files changed

Lines changed: 243 additions & 150 deletions

File tree

‎cli/analyze.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ func mainAnalyze(ctx *cli.Context) error {
155155
final = *aggregate.Live(opCh, nil, "", nil)
156156
}
157157
// If -web is specified, spawn web UI
158+
final.Final = true
158159
monitor.UpdateAggregate(&final, "")
159160
if ctx.Bool("web") {
160161
srv := wui.New(&final)

‎cli/merge.go‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package cli
1919

2020
import (
21+
"encoding/json"
2122
"errors"
2223
"fmt"
2324
"os"
@@ -27,6 +28,7 @@ import (
2728
"github.com/minio/cli"
2829
"github.com/minio/mc/pkg/probe"
2930
"github.com/minio/pkg/v3/console"
31+
"github.com/minio/warp/pkg/aggregate"
3032
"github.com/minio/warp/pkg/bench"
3133
)
3234

@@ -48,21 +50,83 @@ var mergeCmd = cli.Command{
4850
{{.HelpName}} - {{.Usage}}
4951
5052
USAGE:
51-
{{.HelpName}} [FLAGS] benchmark-data-file1 benchmark-data-file2 ...
53+
{{.HelpName}} [FLAGS] benchmark-data-file1 benchmark-data-file2 ...
5254
-> see https://github.com/minio/warp#merging-benchmarks
5355
5456
FLAGS:
5557
{{range .VisibleFlags}}{{.}}
5658
{{end}}`,
5759
}
5860

59-
// mainAnalyze is the entry point for analyze command.
6061
func mainMerge(ctx *cli.Context) error {
6162
checkMerge(ctx)
6263
args := ctx.Args()
6364
if len(args) <= 1 {
6465
console.Fatal("Two or more benchmark data files must be supplied")
6566
}
67+
68+
rc, isJSON := openInput(args[0])
69+
rc.Close()
70+
71+
if isJSON {
72+
return mergeJSON(ctx, args)
73+
}
74+
return mergeCSV(ctx, args)
75+
}
76+
77+
func timeOverlaps(a, b aggregate.LiveAggregate) bool {
78+
return a.StartTime.Before(b.EndTime) && b.StartTime.Before(a.EndTime)
79+
}
80+
81+
func mergeJSON(ctx *cli.Context, args []string) error {
82+
var merged aggregate.Realtime
83+
for i, arg := range args {
84+
rc, isJSON := openInput(arg)
85+
if !isJSON {
86+
rc.Close()
87+
fatalIf(probe.NewError(errors.New("mixed input types")), "mixed input types (JSON and CSV)")
88+
}
89+
var rt aggregate.Realtime
90+
if err := json.NewDecoder(rc).Decode(&rt); err != nil {
91+
rc.Close()
92+
fatalIf(probe.NewError(err), "Unable to parse input")
93+
}
94+
rc.Close()
95+
if i > 0 && !timeOverlaps(merged.Total, rt.Total) {
96+
fatalIf(probe.NewError(fmt.Errorf(
97+
"file %q (%s - %s) does not overlap with merged range (%s - %s)",
98+
arg, rt.Total.StartTime.Format(time.RFC3339), rt.Total.EndTime.Format(time.RFC3339),
99+
merged.Total.StartTime.Format(time.RFC3339), merged.Total.EndTime.Format(time.RFC3339),
100+
)), "time ranges do not overlap")
101+
}
102+
merged.Merge(&rt)
103+
}
104+
merged.Final = true
105+
106+
fileName := ctx.String("benchdata")
107+
if fileName == "" {
108+
fileName = fmt.Sprintf("%s-%s-%s", appName, ctx.Command.Name, time.Now().Format("2006-01-02[150405]"))
109+
}
110+
f, err := os.Create(fileName + ".json.zst")
111+
if err != nil {
112+
console.Error("Unable to write benchmark data:", err)
113+
return nil
114+
}
115+
defer f.Close()
116+
enc, err := zstd.NewWriter(f, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
117+
fatalIf(probe.NewError(err), "Unable to compress benchmark output")
118+
defer enc.Close()
119+
120+
js := json.NewEncoder(enc)
121+
js.SetIndent("", " ")
122+
err = js.Encode(merged)
123+
fatalIf(probe.NewError(err), "Unable to write benchmark output")
124+
125+
console.Infof("Benchmark data written to %q\n", fileName+".json.zst")
126+
return nil
127+
}
128+
129+
func mergeCSV(ctx *cli.Context, args []string) error {
66130
zstdDec, _ := zstd.NewReader(nil)
67131
defer zstdDec.Close()
68132
var allOps bench.Operations
@@ -72,6 +136,13 @@ func mainMerge(ctx *cli.Context) error {
72136
log = nil
73137
}
74138
for _, arg := range args {
139+
rc, isJSON := openInput(arg)
140+
if isJSON {
141+
rc.Close()
142+
fatalIf(probe.NewError(errors.New("mixed input types")), "mixed input types (JSON and CSV)")
143+
}
144+
rc.Close()
145+
75146
f, err := os.Open(arg)
76147
fatalIf(probe.NewError(err), "Unable to open input file")
77148
defer f.Close()

‎go.mod‎

Lines changed: 43 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
module github.com/minio/warp
22

3-
go 1.25.5
3+
go 1.25.0
4+
5+
toolchain go1.25.7
46

57
tool (
68
github.com/golangci/golangci-lint/v2/cmd/golangci-lint
79
mvdan.cc/gofumpt
810
)
911

1012
require (
11-
cloud.google.com/go/storage v1.59.1
12-
github.com/apache/arrow-go/v18 v18.5.0
13-
github.com/apache/iceberg-go v0.4.1-0.20260123180146-3ce3b4bb51c0
13+
cloud.google.com/go/storage v1.59.2
14+
github.com/apache/arrow-go/v18 v18.5.1
15+
github.com/apache/iceberg-go v0.4.1-0.20260210180002-a33fb54d81a1
1416
github.com/aws/aws-sdk-go-v2 v1.41.1
1517
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
1618
github.com/bygui86/multi-profile/v2 v2.1.0
@@ -22,21 +24,21 @@ require (
2224
github.com/fatih/color v1.18.0
2325
github.com/influxdata/influxdb-client-go/v2 v2.14.0
2426
github.com/jfsmig/prng v0.0.2
25-
github.com/klauspost/compress v1.18.2
27+
github.com/klauspost/compress v1.18.4
2628
github.com/minio/cli v1.24.2
27-
github.com/minio/madmin-go/v4 v4.10.0
29+
github.com/minio/madmin-go/v4 v4.6.3
2830
github.com/minio/mc v0.0.0-20251106162529-77f82e18b540
2931
github.com/minio/md5-simd v1.1.2
30-
github.com/minio/minio-go/v7 v7.0.97
31-
github.com/minio/pkg/v3 v3.6.0
32+
github.com/minio/minio-go/v7 v7.0.98
33+
github.com/minio/pkg/v3 v3.6.1
3234
github.com/minio/websocket v1.6.0
3335
github.com/muesli/termenv v0.16.0
3436
github.com/posener/complete v1.2.3
3537
gitlab.com/go-extension/http v0.0.0-20251006175957-3113a1e48242
3638
gitlab.com/go-extension/tls v0.0.0-20251218095625-a161feb36b6b
3739
golang.org/x/sync v0.19.0
3840
golang.org/x/time v0.14.0
39-
google.golang.org/api v0.260.0
41+
google.golang.org/api v0.265.0
4042
gopkg.in/yaml.v3 v3.0.1
4143
)
4244

@@ -48,7 +50,7 @@ require (
4850
atomicgo.dev/schedule v0.1.0 // indirect
4951
cel.dev/expr v0.24.0 // indirect
5052
cloud.google.com/go v0.123.0 // indirect
51-
cloud.google.com/go/auth v0.18.0 // indirect
53+
cloud.google.com/go/auth v0.18.1 // indirect
5254
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
5355
cloud.google.com/go/compute/metadata v0.9.0 // indirect
5456
cloud.google.com/go/iam v1.5.3 // indirect
@@ -104,7 +106,7 @@ require (
104106
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
105107
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
106108
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
107-
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 // indirect
109+
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
108110
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
109111
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
110112
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
@@ -149,8 +151,8 @@ require (
149151
github.com/dlclark/regexp2 v1.11.5 // indirect
150152
github.com/ebitengine/purego v0.9.1 // indirect
151153
github.com/emmansun/gmsm v0.40.0 // indirect
152-
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
153-
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
154+
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
155+
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
154156
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
155157
github.com/ettle/strcase v0.2.0 // indirect
156158
github.com/fatih/structtag v1.2.0 // indirect
@@ -195,12 +197,12 @@ require (
195197
github.com/golangci/revgrep v0.8.0 // indirect
196198
github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect
197199
github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect
198-
github.com/google/flatbuffers v25.9.23+incompatible // indirect
200+
github.com/google/flatbuffers v25.12.19+incompatible // indirect
199201
github.com/google/go-cmp v0.7.0 // indirect
200202
github.com/google/s2a-go v0.1.9 // indirect
201203
github.com/google/uuid v1.6.0 // indirect
202204
github.com/google/wire v0.7.0 // indirect
203-
github.com/googleapis/enterprise-certificate-proxy v0.3.9 // indirect
205+
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
204206
github.com/googleapis/gax-go/v2 v2.16.0 // indirect
205207
github.com/gookit/color v1.5.4 // indirect
206208
github.com/gordonklaus/ineffassign v0.2.0 // indirect
@@ -212,7 +214,7 @@ require (
212214
github.com/hashicorp/errwrap v1.1.0 // indirect
213215
github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect
214216
github.com/hashicorp/go-multierror v1.1.1 // indirect
215-
github.com/hashicorp/go-version v1.7.0 // indirect
217+
github.com/hashicorp/go-version v1.8.0 // indirect
216218
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
217219
github.com/hashicorp/hcl v1.0.0 // indirect
218220
github.com/hexops/gotextdiff v1.0.3 // indirect
@@ -280,7 +282,7 @@ require (
280282
github.com/pelletier/go-toml v1.9.5 // indirect
281283
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
282284
github.com/philhofer/fwd v1.2.0 // indirect
283-
github.com/pierrec/lz4/v4 v4.1.22 // indirect
285+
github.com/pierrec/lz4/v4 v4.1.23 // indirect
284286
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
285287
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
286288
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -289,10 +291,10 @@ require (
289291
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
290292
github.com/prometheus/client_golang v1.23.2 // indirect
291293
github.com/prometheus/client_model v0.6.2 // indirect
292-
github.com/prometheus/common v0.67.4 // indirect
294+
github.com/prometheus/common v0.67.5 // indirect
293295
github.com/prometheus/procfs v0.19.2 // indirect
294296
github.com/prometheus/prom2json v1.5.0 // indirect
295-
github.com/prometheus/prometheus v0.308.1 // indirect
297+
github.com/prometheus/prometheus v0.309.1 // indirect
296298
github.com/pterm/pterm v0.12.82 // indirect
297299
github.com/quasilyte/go-ruleguard v0.4.4 // indirect
298300
github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect
@@ -314,7 +316,7 @@ require (
314316
github.com/secure-io/sio-go v0.3.1 // indirect
315317
github.com/securego/gosec/v2 v2.22.8 // indirect
316318
github.com/segmentio/asm v1.2.1 // indirect
317-
github.com/shirou/gopsutil/v4 v4.25.11 // indirect
319+
github.com/shirou/gopsutil/v4 v4.26.1 // indirect
318320
github.com/sirupsen/logrus v1.9.3 // indirect
319321
github.com/sivchari/containedctx v1.0.3 // indirect
320322
github.com/sonatard/noctx v0.4.0 // indirect
@@ -331,13 +333,13 @@ require (
331333
github.com/stretchr/objx v0.5.2 // indirect
332334
github.com/stretchr/testify v1.11.1 // indirect
333335
github.com/subosito/gotenv v1.4.1 // indirect
334-
github.com/substrait-io/substrait v0.78.1 // indirect
335-
github.com/substrait-io/substrait-go/v7 v7.2.2 // indirect
336-
github.com/substrait-io/substrait-protobuf/go v0.78.1 // indirect
336+
github.com/substrait-io/substrait v0.79.0 // indirect
337+
github.com/substrait-io/substrait-go/v7 v7.3.0 // indirect
338+
github.com/substrait-io/substrait-protobuf/go v0.79.0 // indirect
337339
github.com/tetafro/godot v1.5.4 // indirect
338340
github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect
339341
github.com/timonwong/loggercheck v0.11.0 // indirect
340-
github.com/tinylib/msgp v1.6.1 // indirect
342+
github.com/tinylib/msgp v1.6.3 // indirect
341343
github.com/tklauser/go-sysconf v0.3.16 // indirect
342344
github.com/tklauser/numcpus v0.11.0 // indirect
343345
github.com/tomarrell/wrapcheck/v2 v2.11.0 // indirect
@@ -368,32 +370,33 @@ require (
368370
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
369371
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
370372
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
371-
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
372-
go.opentelemetry.io/otel v1.38.0 // indirect
373-
go.opentelemetry.io/otel/metric v1.38.0 // indirect
374-
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
375-
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
376-
go.opentelemetry.io/otel/trace v1.38.0 // indirect
373+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
374+
go.opentelemetry.io/otel v1.39.0 // indirect
375+
go.opentelemetry.io/otel/metric v1.39.0 // indirect
376+
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
377+
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
378+
go.opentelemetry.io/otel/trace v1.39.0 // indirect
377379
go.uber.org/automaxprocs v1.6.0 // indirect
378380
go.uber.org/multierr v1.11.0 // indirect
379381
go.uber.org/zap v1.27.1 // indirect
380382
go.yaml.in/yaml/v2 v2.4.3 // indirect
383+
go.yaml.in/yaml/v3 v3.0.4 // indirect
381384
gocloud.dev v0.44.0 // indirect
382-
golang.org/x/crypto v0.46.0 // indirect
385+
golang.org/x/crypto v0.48.0 // indirect
383386
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 // indirect
384387
golang.org/x/exp/typeparams v0.0.0-20250911091902-df9299821621 // indirect
385-
golang.org/x/mod v0.31.0 // indirect
386-
golang.org/x/net v0.48.0 // indirect
388+
golang.org/x/mod v0.32.0 // indirect
389+
golang.org/x/net v0.50.0 // indirect
387390
golang.org/x/oauth2 v0.34.0 // indirect
388-
golang.org/x/sys v0.39.0 // indirect
389-
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect
390-
golang.org/x/term v0.38.0 // indirect
391-
golang.org/x/text v0.32.0 // indirect
392-
golang.org/x/tools v0.40.0 // indirect
391+
golang.org/x/sys v0.41.0 // indirect
392+
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect
393+
golang.org/x/term v0.40.0 // indirect
394+
golang.org/x/text v0.34.0 // indirect
395+
golang.org/x/tools v0.41.0 // indirect
393396
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
394397
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
395-
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
396-
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
398+
google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect
399+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
397400
google.golang.org/grpc v1.78.0 // indirect
398401
google.golang.org/protobuf v1.36.11 // indirect
399402
gopkg.in/ini.v1 v1.67.0 // indirect

0 commit comments

Comments
 (0)