Merge pull request #24 from questdb/vi_pgx

chore: default questsb to use pgx for query benchmark

Author: bluestreak01

.github/workflows/go.yml

@@ -13,9 +13,9 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.20'
+        go-version: '1.23'
 
     - name: Test
       run: go test -v -race ./...

.gitignore

@@ -8,3 +8,6 @@
 
 # High Dynamic Range (HDR) Histogram files
 *.hdr
+
+# Compiled binaries
+tsbs_*

.travis.yml

@@ -3,16 +3,9 @@ dist: focal
 jobs:
   include:
     - stage: test
-      name: "Go 1.14"
+      name: "Go 1.23"
       go:
-        - 1.14.x
+        - 1.23.x
       install: skip
       script:
-        - GO111MODULE=on go test -v -race -coverprofile=coverage.txt -covermode=atomic ./...
-    - stage: test
-      name: "Go 1.15"
-      go:
-        - 1.15.x
-      install: skip
-      script:
-        - GO111MODULE=on go test -v -race -coverprofile=coverage.txt -covermode=atomic ./...
+        - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./...

cmd/tsbs_generate_queries/databases/questdb/common.go

@@ -1,8 +1,10 @@
 package questdb
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/questdb/tsbs/cmd/tsbs_generate_queries/uses/devops"
@@ -19,7 +21,7 @@ func (g *BaseGenerator) GenerateEmptyQuery() query.Query {
 	return query.NewHTTP()
 }
 
-// fillInQuery fills the query struct with data.
+// fillInQuery fills the query struct with data (legacy non-parameterized).
 func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql string) {
 	v := url.Values{}
 	v.Set("count", "false")
@@ -33,6 +35,55 @@ func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, sql s
 	q.Body = nil
 }
 
+// fillInQueryWithParams fills the query struct with parameterized SQL and bind values.
+// sqlTemplate uses $1, $2, etc. placeholders for PostgreSQL prepared statements.
+// params contains the values to bind at runtime.
+func (g *BaseGenerator) fillInQueryWithParams(qi query.Query, humanLabel, humanDesc, sqlTemplate string, params []interface{}) {
+	q := qi.(*query.HTTP)
+	q.HumanLabel = []byte(humanLabel)
+	q.HumanDescription = []byte(humanDesc)
+	q.Method = []byte("GET")
+
+	// Store parameterized SQL in RawQuery
+	q.RawQuery = []byte(sqlTemplate)
+
+	// Store parameters as JSON in Body for pgx mode
+	paramsJSON, _ := json.Marshal(params)
+	q.Body = paramsJSON
+
+	// For HTTP mode, we still need to generate the full SQL path
+	// Substitute parameters to create the HTTP URL
+	fullSQL := substituteParams(sqlTemplate, params)
+	v := url.Values{}
+	v.Set("count", "false")
+	v.Set("query", fullSQL)
+	q.Path = []byte(fmt.Sprintf("/exec?%s", v.Encode()))
+}
+
+// substituteParams replaces $1, $2, etc. with actual values for HTTP mode
+func substituteParams(sql string, params []interface{}) string {
+	result := sql
+	for i, param := range params {
+		placeholder := fmt.Sprintf("$%d", i+1)
+		var replacement string
+		switch v := param.(type) {
+		case string:
+			replacement = fmt.Sprintf("'%s'", v)
+		case []string:
+			// Handle string arrays for hostname IN clauses
+			quoted := make([]string, len(v))
+			for j, s := range v {
+				quoted[j] = fmt.Sprintf("'%s'", s)
+			}
+			replacement = fmt.Sprintf("(%s)", strings.Join(quoted, ", "))
+		default:
+			replacement = fmt.Sprintf("%v", v)
+		}
+		result = strings.Replace(result, placeholder, replacement, 1)
+	}
+	return result
+}
+
 // NewDevops creates a new devops use case query generator.
 func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) {
 	core, err := devops.NewCore(start, end, scale)

cmd/tsbs_generate_queries/databases/questdb/devops.go

@@ -49,24 +49,26 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) {
 	hosts, err := d.GetRandomHosts(nHosts)
 	panicIfErr(err)
 
-	sql := fmt.Sprintf(`
+	// Parameterized SQL for prepared statements
+	sqlTemplate := fmt.Sprintf(`
 		SELECT
-			date_trunc('hour', timestamp) AS hour,
 			%s
 		FROM cpu
-		WHERE hostname IN ('%s')
-		  AND timestamp >= '%s'
-		  AND timestamp < '%s'
-		GROUP BY hour
-		ORDER BY hour`,
-		strings.Join(selectClauses, ", "),
-		strings.Join(hosts, "', '"),
+		WHERE hostname IN $1
+		  AND timestamp >= $2
+		  AND timestamp < $3
+		SAMPLE BY 1h`,
+		strings.Join(selectClauses, ", "))
+
+	params := []interface{}{
+		hosts,
 		interval.StartString(),
-		interval.EndString())
+		interval.EndString(),
+	}
 
 	humanLabel := devops.GetMaxAllLabel("QuestDB", nHosts)
 	humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
-	d.fillInQuery(qi, humanLabel, humanDesc, sql)
+	d.fillInQueryWithParams(qi, humanLabel, humanDesc, sqlTemplate, params)
 }
 
 // GroupByTimeAndPrimaryTag selects the AVG of metrics in the group `cpu` per device
@@ -82,21 +84,24 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
 	interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)
 	selectClauses := d.getSelectAggClauses("avg", metrics)
 
-	sql := fmt.Sprintf(`
-		SELECT date_trunc('hour', timestamp) as timestamp, hostname,
+	// Parameterized SQL for prepared statements
+	sqlTemplate := fmt.Sprintf(`
+		SELECT hostname,
 			%s
 		FROM cpu
-		WHERE timestamp >= '%s'
-		  AND timestamp < '%s'
-		GROUP BY timestamp, hostname
-		ORDER BY timestamp, hostname`,
-		strings.Join(selectClauses, ", "),
+		WHERE timestamp >= $1
+		  AND timestamp < $2
+		SAMPLE BY 1h`,
+		strings.Join(selectClauses, ", "))
+
+	params := []interface{}{
 		interval.StartString(),
-		interval.EndString())
+		interval.EndString(),
+	}
 
 	humanLabel := devops.GetDoubleGroupByLabel("QuestDB", numMetrics)
 	humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
-	d.fillInQuery(qi, humanLabel, humanDesc, sql)
+	d.fillInQueryWithParams(qi, humanLabel, humanDesc, sqlTemplate, params)
 }
 
 // GroupByOrderByLimit populates a query.Query that has a time WHERE clause,
@@ -106,19 +111,23 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
 // groupby-orderby-limit
 func (d *Devops) GroupByOrderByLimit(qi query.Query) {
 	interval := d.Interval.MustRandWindow(time.Hour)
-	sql := fmt.Sprintf(`
-		SELECT date_trunc('minute', timestamp) AS minute,
-			max(usage_user)
+
+	// Parameterized SQL for prepared statements
+	sqlTemplate := `
+		SELECT max(usage_user)
 		FROM cpu
-		WHERE timestamp < '%s'
-		GROUP BY minute
-		ORDER BY minute DESC
-		LIMIT 5`,
-		interval.EndString())
+		WHERE timestamp < $1
+		SAMPLE BY 1m
+		ORDER BY timestamp DESC
+		LIMIT 5`
+
+	params := []interface{}{
+		interval.EndString(),
+	}
 
 	humanLabel := "QuestDB max cpu over last 5 min-intervals (random end)"
 	humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.EndString())
-	d.fillInQuery(qi, humanLabel, humanDesc, sql)
+	d.fillInQueryWithParams(qi, humanLabel, humanDesc, sqlTemplate, params)
 }
 
 // LastPointPerHost finds the last row for every host in the dataset
@@ -142,36 +151,45 @@ func (d *Devops) LastPointPerHost(qi query.Query) {
 // high-cpu-all
 func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
 	interval := d.Interval.MustRandWindow(devops.HighCPUDuration)
-	sql := ""
+
+	var sqlTemplate string
+	var params []interface{}
+
 	if nHosts > 0 {
 		hosts, err := d.GetRandomHosts(nHosts)
 		panicIfErr(err)
 
-		sql = fmt.Sprintf(`
+		// Parameterized SQL for prepared statements
+		sqlTemplate = `
 		      SELECT *
 		      FROM cpu
 		      WHERE usage_user > 90.0
-		       AND hostname IN ('%s')
-		       AND timestamp >= '%s'
-		       AND timestamp < '%s'`,
-			strings.Join(hosts, "', '"),
+		       AND hostname IN $1
+		       AND timestamp >= $2
+		       AND timestamp < $3`
+		params = []interface{}{
+			hosts,
 			interval.StartString(),
-			interval.EndString())
+			interval.EndString(),
+		}
 	} else {
-		sql = fmt.Sprintf(`
+		// Parameterized SQL for prepared statements (no hostname filter)
+		sqlTemplate = `
 		      SELECT *
 		      FROM cpu
 		      WHERE usage_user > 90.0
-		       AND timestamp >= '%s'
-		       AND timestamp < '%s'`,
+		       AND timestamp >= $1
+		       AND timestamp < $2`
+		params = []interface{}{
 			interval.StartString(),
-			interval.EndString())
+			interval.EndString(),
+		}
 	}
 
 	humanLabel, err := devops.GetHighCPULabel("QuestDB", nHosts)
 	panicIfErr(err)
 	humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
-	d.fillInQuery(qi, humanLabel, humanDesc, sql)
+	d.fillInQueryWithParams(qi, humanLabel, humanDesc, sqlTemplate, params)
 }
 
 // GroupByTime selects the MAX for metrics under 'cpu', per minute for N random
@@ -192,23 +210,26 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t
 	hosts, err := d.GetRandomHosts(nHosts)
 	panicIfErr(err)
 
-	sql := fmt.Sprintf(`
-		SELECT date_trunc('minute', timestamp) as minute,
+	// Parameterized SQL for prepared statements
+	sqlTemplate := fmt.Sprintf(`
+		SELECT
 			%s
 		FROM cpu
-		WHERE hostname IN ('%s')
-		  AND timestamp >= '%s'
-		  AND timestamp < '%s'
-		GROUP BY minute
-		ORDER BY minute`,
-		strings.Join(selectClauses, ", "),
-		strings.Join(hosts, "', '"),
+		WHERE hostname IN $1
+		  AND timestamp >= $2
+		  AND timestamp < $3
+		SAMPLE BY 1m`,
+		strings.Join(selectClauses, ", "))
+
+	params := []interface{}{
+		hosts,
 		interval.StartString(),
-		interval.EndString())
+		interval.EndString(),
+	}
 
 	humanLabel := fmt.Sprintf(
 		"QuestDB %d cpu metric(s), random %4d hosts, random %s by 1m",
 		numMetrics, nHosts, timeRange)
 	humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
-	d.fillInQuery(qi, humanLabel, humanDesc, sql)
+	d.fillInQueryWithParams(qi, humanLabel, humanDesc, sqlTemplate, params)
 }

cmd/tsbs_generate_queries/databases/questdb/devops_test.go

@@ -14,8 +14,8 @@ import (
 func TestDevopsGroupByTime(t *testing.T) {
 	expectedHumanLabel := "QuestDB 1 cpu metric(s), random    1 hosts, random 1s by 1m"
 	expectedHumanDesc := "QuestDB 1 cpu metric(s), random    1 hosts, random 1s by 1m: 1970-01-01T00:05:58Z"
-	expectedQuery := "SELECT date_trunc('minute', timestamp) as minute, max(usage_user) AS max_usage_user FROM cpu " +
-		"WHERE hostname IN ('host_9') AND timestamp >= '1970-01-01T00:05:58Z' AND timestamp < '1970-01-01T00:05:59Z' GROUP BY minute ORDER BY minute"
+	expectedQuery := "SELECT max(usage_user) AS max_usage_user FROM cpu " +
+		"WHERE hostname IN ('host_9') AND timestamp >= '1970-01-01T00:05:58Z' AND timestamp < '1970-01-01T00:05:59Z' SAMPLE BY 1m"
 
 	rand.Seed(123) // Setting seed for testing purposes.
 	s := time.Unix(0, 0)
@@ -40,8 +40,8 @@ func TestDevopsGroupByTime(t *testing.T) {
 func TestDevopsGroupByOrderByLimit(t *testing.T) {
 	expectedHumanLabel := "QuestDB max cpu over last 5 min-intervals (random end)"
 	expectedHumanDesc := "QuestDB max cpu over last 5 min-intervals (random end): 1970-01-01T01:16:22Z"
-	expectedQuery := "SELECT date_trunc('minute', timestamp) AS minute, max(usage_user) FROM cpu " +
-		"WHERE timestamp < '1970-01-01T01:16:22Z' GROUP BY minute ORDER BY minute DESC LIMIT 5"
+	expectedQuery := "SELECT max(usage_user) FROM cpu " +
+		"WHERE timestamp < '1970-01-01T01:16:22Z' SAMPLE BY 1m ORDER BY timestamp DESC LIMIT 5"
 
 	rand.Seed(123) // Setting seed for testing purposes.
 	s := time.Unix(0, 0)
@@ -72,18 +72,18 @@ func TestDevopsGroupByTimeAndPrimaryTag(t *testing.T) {
 			input:              1,
 			expectedHumanLabel: "QuestDB mean of 1 metrics, all hosts, random 12h0m0s by 1h",
 			expectedHumanDesc:  "QuestDB mean of 1 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:16:22Z",
-			expectedQuery: "SELECT date_trunc('hour', timestamp) as timestamp, hostname, avg(usage_user) AS avg_usage_user FROM cpu " +
+			expectedQuery: "SELECT hostname, avg(usage_user) AS avg_usage_user FROM cpu " +
 				"WHERE timestamp >= '1970-01-01T00:16:22Z' AND timestamp < '1970-01-01T12:16:22Z' " +
-				"GROUP BY timestamp, hostname ORDER BY timestamp, hostname",
+				"SAMPLE BY 1h",
 		},
 		{
 			desc:               "5 metrics",
 			input:              5,
 			expectedHumanLabel: "QuestDB mean of 5 metrics, all hosts, random 12h0m0s by 1h",
 			expectedHumanDesc:  "QuestDB mean of 5 metrics, all hosts, random 12h0m0s by 1h: 1970-01-01T00:54:10Z",
-			expectedQuery: "SELECT date_trunc('hour', timestamp) as timestamp, hostname, avg(usage_user) AS avg_usage_user, avg(usage_system) AS avg_usage_system, avg(usage_idle) AS avg_usage_idle, avg(usage_nice) AS avg_usage_nice, avg(usage_iowait) AS avg_usage_iowait FROM cpu " +
+			expectedQuery: "SELECT hostname, avg(usage_user) AS avg_usage_user, avg(usage_system) AS avg_usage_system, avg(usage_idle) AS avg_usage_idle, avg(usage_nice) AS avg_usage_nice, avg(usage_iowait) AS avg_usage_iowait FROM cpu " +
 				"WHERE timestamp >= '1970-01-01T00:54:10Z' AND timestamp < '1970-01-01T12:54:10Z' " +
-				"GROUP BY timestamp, hostname ORDER BY timestamp, hostname",
+				"SAMPLE BY 1h",
 		},
 	}
 
@@ -112,16 +112,16 @@ func TestMaxAllCPU(t *testing.T) {
 			input:              1,
 			expectedHumanLabel: "QuestDB max of all CPU metrics, random    1 hosts, random 8h0m0s by 1h",
 			expectedHumanDesc:  "QuestDB max of all CPU metrics, random    1 hosts, random 8h0m0s by 1h: 1970-01-01T00:54:10Z",
-			expectedQuery: "SELECT date_trunc('hour', timestamp) AS hour, max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu " +
+			expectedQuery: "SELECT max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu " +
 				"WHERE hostname IN ('host_3') AND timestamp >= '1970-01-01T00:54:10Z' AND timestamp < '1970-01-01T08:54:10Z' " +
-				"GROUP BY hour ORDER BY hour",
+				"SAMPLE BY 1h",
 		},
 		{
 			desc:               "5 hosts",
 			input:              5,
 			expectedHumanLabel: "QuestDB max of all CPU metrics, random    5 hosts, random 8h0m0s by 1h",
 			expectedHumanDesc:  "QuestDB max of all CPU metrics, random    5 hosts, random 8h0m0s by 1h: 1970-01-01T00:37:12Z",
-			expectedQuery:      "SELECT date_trunc('hour', timestamp) AS hour, max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu WHERE hostname IN ('host_9', 'host_5', 'host_1', 'host_7', 'host_2') AND timestamp >= '1970-01-01T00:37:12Z' AND timestamp < '1970-01-01T08:37:12Z' GROUP BY hour ORDER BY hour",
+			expectedQuery:      "SELECT max(usage_user) AS max_usage_user, max(usage_system) AS max_usage_system, max(usage_idle) AS max_usage_idle, max(usage_nice) AS max_usage_nice, max(usage_iowait) AS max_usage_iowait, max(usage_irq) AS max_usage_irq, max(usage_softirq) AS max_usage_softirq, max(usage_steal) AS max_usage_steal, max(usage_guest) AS max_usage_guest, max(usage_guest_nice) AS max_usage_guest_nice FROM cpu WHERE hostname IN ('host_9', 'host_5', 'host_1', 'host_7', 'host_2') AND timestamp >= '1970-01-01T00:37:12Z' AND timestamp < '1970-01-01T08:37:12Z' SAMPLE BY 1h",
 		},
 	}