Skip to content

Commit 1eb7705

Browse files
patrickSpaceSurferbsmthbluestreak01
authored
Questdb benchmark support (#157)
Co-authored-by: bsmth <[email protected]> Co-authored-by: Vlad Ilyushchenko <[email protected]>
1 parent 161c39d commit 1eb7705

File tree

25 files changed

+1848
-2
lines changed

25 files changed

+1848
-2
lines changed

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
.DS_Store
44
.idea
55
.vscode
6+
*~
67

78
# High Dynamic Range (HDR) Histogram files
8-
*.hdr
9+
*.hdr

‎README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ Current databases supported:
1212
+ CrateDB [(supplemental docs)](docs/cratedb.md)
1313
+ InfluxDB [(supplemental docs)](docs/influx.md)
1414
+ MongoDB [(supplemental docs)](docs/mongo.md)
15+
+ QuestDB [(supplemental docs)](docs/questdb.md)
1516
+ SiriDB [(supplemental docs)](docs/siridb.md)
1617
+ TimescaleDB [(supplemental docs)](docs/timescaledb.md)
1718
+ Timestream [(supplemental docs)](docs/timestream.md)
@@ -75,6 +76,7 @@ cases are implemented for each database:
7576
|CrateDB|X||
7677
|InfluxDB|X|X|
7778
|MongoDB|X|
79+
|QuestDB|X|X
7880
|SiriDB|X|
7981
|TimescaleDB|X|X|
8082
|Timestream|X||
@@ -132,7 +134,7 @@ Variables needed:
132134
1. an end time. E.g., `2016-01-04T00:00:00Z`
133135
1. how much time should be between each reading per device, in seconds. E.g., `10s`
134136
1. and which database(s) you want to generate for. E.g., `timescaledb`
135-
(choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `siridb`,
137+
(choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `questdb`, `siridb`,
136138
`timescaledb` or `victoriametrics`)
137139

138140
Given the above steps you can now generate a dataset (or multiple

‎cmd/tsbs_generate_queries/databases/questdb/common.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package questdb
2+
3+
import (
4+
"fmt"
5+
"net/url"
6+
"time"
7+
8+
"github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops"
9+
"github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils"
10+
"github.com/timescale/tsbs/pkg/query"
11+
)
12+
13+
// BaseGenerator contains settings specific for QuestDB
14+
type BaseGenerator struct {
15+
}
16+
17+
// GenerateEmptyQuery returns an empty query.QuestDB.
18+
func (g *BaseGenerator) GenerateEmptyQuery() query.Query {
19+
return query.NewHTTP()
20+
}
21+
22+
// fillInQuery fills the query struct with data.
23+
func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql string) {
24+
v := url.Values{}
25+
v.Set("count", "false")
26+
v.Set("query", sql)
27+
q := qi.(*query.HTTP)
28+
q.HumanLabel = []byte(humanLabel)
29+
q.RawQuery = []byte(sql)
30+
q.HumanDescription = []byte(humanDesc)
31+
q.Method = []byte("GET")
32+
q.Path = []byte(fmt.Sprintf("/exec?%s", v.Encode()))
33+
q.Body = nil
34+
}
35+
36+
// NewDevops creates a new devops use case query generator.
37+
func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) {
38+
core, err := devops.NewCore(start, end, scale)
39+
40+
if err != nil {
41+
return nil, err
42+
}
43+
44+
devops := &Devops{
45+
BaseGenerator: g,
46+
Core: core,
47+
}
48+
49+
return devops, nil
50+
}

‎cmd/tsbs_generate_queries/databases/questdb/devops.go

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package questdb
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"time"
7+
8+
"github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops"
9+
"github.com/timescale/tsbs/pkg/query"
10+
)
11+
12+
// TODO: Remove the need for this by continuing to bubble up errors
13+
func panicIfErr(err error) {
14+
if err != nil {
15+
panic(err.Error())
16+
}
17+
}
18+
19+
// Devops produces QuestDB-specific queries for all the devops query types.
20+
type Devops struct {
21+
*BaseGenerator
22+
*devops.Core
23+
}
24+
25+
// getSelectAggClauses builds specified aggregate function clauses for
26+
// a set of column idents.
27+
//
28+
// For instance:
29+
// max(cpu_time) AS max_cpu_time
30+
func (d *Devops) getSelectAggClauses(aggFunc string, idents []string) []string {
31+
selectAggClauses := make([]string, len(idents))
32+
for i, ident := range idents {
33+
selectAggClauses[i] =
34+
fmt.Sprintf("%[1]s(%[2]s) AS %[1]s_%[2]s", aggFunc, ident)
35+
}
36+
return selectAggClauses
37+
}
38+
39+
// MaxAllCPU selects the MAX of all metrics under 'cpu' per hour for N random
40+
// hosts
41+
//
42+
// Queries:
43+
// cpu-max-all-1
44+
// cpu-max-all-8
45+
func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) {
46+
interval := d.Interval.MustRandWindow(devops.MaxAllDuration)
47+
selectClauses := d.getSelectAggClauses("max", devops.GetAllCPUMetrics())
48+
hosts, err := d.GetRandomHosts(nHosts)
49+
panicIfErr(err)
50+
51+
sql := fmt.Sprintf(`
52+
SELECT
53+
hour(timestamp) AS hour,
54+
%s
55+
FROM cpu
56+
WHERE hostname IN ('%s')
57+
AND timestamp >= '%s'
58+
AND timestamp < '%s'
59+
SAMPLE BY 1h`,
60+
strings.Join(selectClauses, ", "),
61+
strings.Join(hosts, "', '"),
62+
interval.StartString(),
63+
interval.EndString())
64+
65+
humanLabel := devops.GetMaxAllLabel("QuestDB", nHosts)
66+
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
67+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
68+
}
69+
70+
// GroupByTimeAndPrimaryTag selects the AVG of metrics in the group `cpu` per device
71+
// per hour for a day
72+
//
73+
// Queries:
74+
// double-groupby-1
75+
// double-groupby-5
76+
// double-groupby-all
77+
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
78+
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
79+
panicIfErr(err)
80+
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)
81+
selectClauses := d.getSelectAggClauses("avg", metrics)
82+
83+
sql := fmt.Sprintf(`
84+
SELECT timestamp, hostname,
85+
%s
86+
FROM cpu
87+
WHERE timestamp >= '%s'
88+
AND timestamp < '%s'
89+
SAMPLE BY 1h
90+
GROUP BY timestamp, hostname`,
91+
strings.Join(selectClauses, ", "),
92+
interval.StartString(),
93+
interval.EndString())
94+
95+
humanLabel := devops.GetDoubleGroupByLabel("QuestDB", numMetrics)
96+
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
97+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
98+
}
99+
100+
// GroupByOrderByLimit populates a query.Query that has a time WHERE clause,
101+
// that groups by a truncated date, orders by that date, and takes a limit:
102+
//
103+
// Queries:
104+
// groupby-orderby-limit
105+
func (d *Devops) GroupByOrderByLimit(qi query.Query) {
106+
interval := d.Interval.MustRandWindow(time.Hour)
107+
sql := fmt.Sprintf(`
108+
SELECT timestamp AS minute,
109+
max(usage_user)
110+
FROM cpu
111+
WHERE timestamp < '%s'
112+
SAMPLE BY 1m
113+
LIMIT 5`,
114+
interval.EndString())
115+
116+
humanLabel := "QuestDB max cpu over last 5 min-intervals (random end)"
117+
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.EndString())
118+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
119+
}
120+
121+
// LastPointPerHost finds the last row for every host in the dataset
122+
//
123+
// Queries:
124+
// lastpoint
125+
func (d *Devops) LastPointPerHost(qi query.Query) {
126+
sql := fmt.Sprintf(`SELECT * FROM cpu latest by hostname`)
127+
128+
humanLabel := "QuestDB last row per host"
129+
humanDesc := humanLabel
130+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
131+
}
132+
133+
// HighCPUForHosts populates a query that gets CPU metrics when the CPU has
134+
// high usage between a time period for a number of hosts (if 0, it will
135+
// search all hosts)
136+
//
137+
// Queries:
138+
// high-cpu-1
139+
// high-cpu-all
140+
func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
141+
interval := d.Interval.MustRandWindow(devops.HighCPUDuration)
142+
sql := ""
143+
if nHosts > 0 {
144+
hosts, err := d.GetRandomHosts(nHosts)
145+
panicIfErr(err)
146+
147+
sql = fmt.Sprintf(`
148+
SELECT *
149+
FROM cpu
150+
WHERE usage_user > 90.0
151+
AND hostname IN ('%s')
152+
AND timestamp >= '%s'
153+
AND timestamp < '%s'`,
154+
strings.Join(hosts, "', '"),
155+
interval.StartString(),
156+
interval.EndString())
157+
} else {
158+
sql = fmt.Sprintf(`
159+
SELECT *
160+
FROM cpu
161+
WHERE usage_user > 90.0
162+
AND timestamp >= '%s'
163+
AND timestamp < '%s'`,
164+
interval.StartString(),
165+
interval.EndString())
166+
}
167+
168+
humanLabel, err := devops.GetHighCPULabel("QuestDB", nHosts)
169+
panicIfErr(err)
170+
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
171+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
172+
}
173+
174+
// GroupByTime selects the MAX for metrics under 'cpu', per minute for N random
175+
// hosts
176+
//
177+
// Resultsets:
178+
// single-groupby-1-1-12
179+
// single-groupby-1-1-1
180+
// single-groupby-1-8-1
181+
// single-groupby-5-1-12
182+
// single-groupby-5-1-1
183+
// single-groupby-5-8-1
184+
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
185+
interval := d.Interval.MustRandWindow(timeRange)
186+
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
187+
panicIfErr(err)
188+
selectClauses := d.getSelectAggClauses("max", metrics)
189+
hosts, err := d.GetRandomHosts(nHosts)
190+
panicIfErr(err)
191+
192+
sql := fmt.Sprintf(`
193+
SELECT timestamp,
194+
%s
195+
FROM cpu
196+
WHERE hostname IN ('%s')
197+
AND timestamp >= '%s'
198+
AND timestamp < '%s'
199+
SAMPLE BY 1m`,
200+
strings.Join(selectClauses, ", "),
201+
strings.Join(hosts, "', '"),
202+
interval.StartString(),
203+
interval.EndString())
204+
205+
humanLabel := fmt.Sprintf(
206+
"QuestDB %d cpu metric(s), random %4d hosts, random %s by 1m",
207+
numMetrics, nHosts, timeRange)
208+
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
209+
d.fillInQuery(qi, humanLabel, humanDesc, sql)
210+
}

0 commit comments

Comments
 (0)