Skip to content

Commit 22939da

Browse files
SlachSlach
authored
Merge pull request #767 from Altinity/issue-742
Add processing of nested group by clauses
2 parents 9aca2d7 + 9553f68 commit 22939da

File tree

5 files changed

+276
-21
lines changed

5 files changed

+276
-21
lines changed

‎pkg/backend_wasm.go

+56-17
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,51 @@ type QueryRequest struct {
214214
}
215215
}
216216

217+
// findGroupByProperties recursively searches for GROUP BY clauses in the AST
218+
func findGroupByProperties(ast *eval.EvalAST) []interface{} {
219+
// First, check if there's a GROUP BY at this level
220+
if prop, exists := ast.Obj["group by"]; exists {
221+
switch v := prop.(type) {
222+
case *eval.EvalAST:
223+
// If the property is an AST object, add all items from its array
224+
properties := make([]interface{}, len(v.Arr))
225+
copy(properties, v.Arr)
226+
return properties
227+
case []interface{}:
228+
// If the property is already a slice, use it directly
229+
return v
230+
default:
231+
// For any other type, add it as a single item
232+
return []interface{}{v}
233+
}
234+
}
235+
236+
// If not found at this level, check if there's a FROM clause that might contain a subquery
237+
if from, exists := ast.Obj["from"]; exists {
238+
switch v := from.(type) {
239+
case *eval.EvalAST:
240+
// If FROM contains another AST (subquery), recursively search in it
241+
subProperties := findGroupByProperties(v)
242+
if len(subProperties) > 0 {
243+
return subProperties
244+
}
245+
}
246+
}
247+
248+
// If nothing found in subqueries, check any other properties that might contain nested ASTs
249+
for _, obj := range ast.Obj {
250+
if subAST, ok := obj.(*eval.EvalAST); ok {
251+
subProperties := findGroupByProperties(subAST)
252+
if len(subProperties) > 0 {
253+
return subProperties
254+
}
255+
}
256+
}
257+
258+
// Return empty slice if nothing found
259+
return []interface{}{}
260+
}
261+
217262
// createQueryWasm is the WebAssembly-compatible function that processes query creation
218263
func createQueryWasm(this js.Value, args []js.Value) interface{} {
219264
// Validate input arguments
@@ -308,22 +353,8 @@ func createQueryWasm(this js.Value, args []js.Value) interface{} {
308353
}
309354
}
310355

311-
// Extract properties from AST
312-
var properties []interface{}
313-
if prop, exists := ast.Obj["group by"]; exists {
314-
switch v := prop.(type) {
315-
case *eval.EvalAST:
316-
// If the property is an AST object, add all items from its array
317-
properties = make([]interface{}, len(v.Arr))
318-
copy(properties, v.Arr)
319-
case []interface{}:
320-
// If the property is already a slice, use it directly
321-
properties = v
322-
default:
323-
// For any other type, add it as a single item
324-
properties = []interface{}{v}
325-
}
326-
}
356+
// Use the recursive function to find GROUP BY properties at any level
357+
properties := findGroupByProperties(ast)
327358

328359
// Return the result
329360
return map[string]interface{}{
@@ -406,7 +437,15 @@ func getAstPropertyWasm(this js.Value, args []js.Value) interface{} {
406437
}
407438
}
408439

409-
// Extract properties from the AST
440+
// Use the recursive function if we're looking for group by
441+
if propertyName == "group by" {
442+
properties := findGroupByProperties(ast)
443+
return map[string]interface{}{
444+
"properties": properties,
445+
}
446+
}
447+
448+
// Standard extraction for other properties
410449
var properties []interface{}
411450
if prop, exists := ast.Obj[propertyName]; exists {
412451
switch v := prop.(type) {

‎query_debugger/debug_query.go

+189
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io/ioutil"
7+
"os"
8+
"time"
9+
10+
"github.com/altinity/clickhouse-grafana/pkg/eval"
11+
)
12+
13+
// QueryRequest represents the structure of the query request
14+
type QueryRequest struct {
15+
RefId string `json:"refId"`
16+
RuleUid string `json:"ruleUid"`
17+
RawQuery bool `json:"rawQuery"`
18+
Query string `json:"query"`
19+
DateTimeCol string `json:"dateTimeColDataType"`
20+
DateCol string `json:"dateColDataType"`
21+
DateTimeType string `json:"dateTimeType"`
22+
Extrapolate bool `json:"extrapolate"`
23+
SkipComments bool `json:"skip_comments"`
24+
AddMetadata bool `json:"add_metadata"`
25+
Format string `json:"format"`
26+
Round string `json:"round"`
27+
IntervalFactor int `json:"intervalFactor"`
28+
Interval string `json:"interval"`
29+
Database string `json:"database"`
30+
Table string `json:"table"`
31+
MaxDataPoints int64 `json:"maxDataPoints"`
32+
FrontendDatasource bool `json:"frontendDatasource"`
33+
UseWindowFuncForMacros bool `json:"useWindowFuncForMacros"`
34+
TimeRange struct {
35+
From string `json:"from"`
36+
To string `json:"to"`
37+
} `json:"timeRange"`
38+
}
39+
40+
// findGroupByProperties recursively searches for GROUP BY clauses in the AST
41+
func findGroupByProperties(ast *eval.EvalAST) []interface{} {
42+
// First, check if there's a GROUP BY at this level
43+
if prop, exists := ast.Obj["group by"]; exists {
44+
switch v := prop.(type) {
45+
case *eval.EvalAST:
46+
// If the property is an AST object, add all items from its array
47+
properties := make([]interface{}, len(v.Arr))
48+
copy(properties, v.Arr)
49+
return properties
50+
case []interface{}:
51+
// If the property is already a slice, use it directly
52+
return v
53+
default:
54+
// For any other type, add it as a single item
55+
return []interface{}{v}
56+
}
57+
}
58+
59+
// If not found at this level, check if there's a FROM clause that might contain a subquery
60+
if from, exists := ast.Obj["from"]; exists {
61+
switch v := from.(type) {
62+
case *eval.EvalAST:
63+
// If FROM contains another AST (subquery), recursively search in it
64+
subProperties := findGroupByProperties(v)
65+
if len(subProperties) > 0 {
66+
return subProperties
67+
}
68+
}
69+
}
70+
71+
// If nothing found in subqueries, check any other properties that might contain nested ASTs
72+
for _, obj := range ast.Obj {
73+
if subAST, ok := obj.(*eval.EvalAST); ok {
74+
subProperties := findGroupByProperties(subAST)
75+
if len(subProperties) > 0 {
76+
return subProperties
77+
}
78+
}
79+
}
80+
81+
// Return empty slice if nothing found
82+
return []interface{}{}
83+
}
84+
85+
// createQuery is the debug version of createQueryWasm
86+
func createQuery(reqData QueryRequest) map[string]interface{} {
87+
// Parse time range
88+
from, err := time.Parse(time.RFC3339, reqData.TimeRange.From)
89+
if err != nil {
90+
return map[string]interface{}{
91+
"error": "Invalid `$from` time: " + err.Error(),
92+
}
93+
}
94+
95+
to, err := time.Parse(time.RFC3339, reqData.TimeRange.To)
96+
if err != nil {
97+
return map[string]interface{}{
98+
"error": "Invalid `$to` time: " + err.Error(),
99+
}
100+
}
101+
102+
// Create eval.EvalQuery
103+
evalQ := eval.EvalQuery{
104+
RefId: reqData.RefId,
105+
RuleUid: reqData.RuleUid,
106+
RawQuery: reqData.RawQuery,
107+
Query: reqData.Query,
108+
DateTimeCol: reqData.DateTimeCol,
109+
DateCol: reqData.DateCol,
110+
DateTimeType: reqData.DateTimeType,
111+
Extrapolate: reqData.Extrapolate,
112+
SkipComments: reqData.SkipComments,
113+
AddMetadata: reqData.AddMetadata,
114+
Format: reqData.Format,
115+
Round: reqData.Round,
116+
IntervalFactor: reqData.IntervalFactor,
117+
Interval: reqData.Interval,
118+
Database: reqData.Database,
119+
Table: reqData.Table,
120+
MaxDataPoints: reqData.MaxDataPoints,
121+
From: from,
122+
To: to,
123+
FrontendDatasource: reqData.FrontendDatasource,
124+
UseWindowFuncForMacros: reqData.UseWindowFuncForMacros,
125+
}
126+
127+
// Apply macros and get AST
128+
sql, err := evalQ.ApplyMacrosAndTimeRangeToQuery()
129+
if err != nil {
130+
return map[string]interface{}{
131+
"error": fmt.Sprintf("Failed to apply macros: %v", err),
132+
}
133+
}
134+
135+
scanner := eval.NewScanner(sql)
136+
ast, err := scanner.ToAST()
137+
if err != nil {
138+
return map[string]interface{}{
139+
"error": fmt.Sprintf("Failed to parse query: %v", err),
140+
}
141+
}
142+
143+
// Use the recursive function to find GROUP BY properties at any level
144+
properties := findGroupByProperties(ast)
145+
146+
// Return the result with detailed information about the AST structure
147+
return map[string]interface{}{
148+
"sql": sql,
149+
"keys": properties,
150+
"debug": map[string]interface{}{
151+
"hasFromClause": ast.HasOwnProperty("from"),
152+
"hasGroupByClause": ast.HasOwnProperty("group by"),
153+
},
154+
}
155+
}
156+
157+
func main() {
158+
if len(os.Args) < 2 {
159+
fmt.Println("Usage: go run debug_query.go <input.json>")
160+
os.Exit(1)
161+
}
162+
163+
// Read input file
164+
data, err := ioutil.ReadFile(os.Args[1])
165+
if err != nil {
166+
fmt.Printf("Error reading file: %v\n", err)
167+
os.Exit(1)
168+
}
169+
170+
// Parse input JSON
171+
var request QueryRequest
172+
err = json.Unmarshal(data, &request)
173+
if err != nil {
174+
fmt.Printf("Error parsing JSON: %v\n", err)
175+
os.Exit(1)
176+
}
177+
178+
// Process the query
179+
result := createQuery(request)
180+
181+
// Print result
182+
resultJSON, err := json.MarshalIndent(result, "", " ")
183+
if err != nil {
184+
fmt.Printf("Error encoding result: %v\n", err)
185+
os.Exit(1)
186+
}
187+
188+
fmt.Println(string(resultJSON))
189+
}

‎query_debugger/input.json

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"frontendDatasource": true,
3+
"refId": "B",
4+
"ruleUid": "",
5+
"rawQuery": false,
6+
"query": "$lttbMs(auto, category, event_time, requests) FROM $table WHERE $timeFilter GROUP BY category",
7+
"dateTimeColDataType": "event_time",
8+
"dateColDataType": "",
9+
"dateTimeType": "DATETIME",
10+
"extrapolate": true,
11+
"skip_comments": true,
12+
"add_metadata": true,
13+
"useWindowFuncForMacros": true,
14+
"format": "time_series",
15+
"round": "0s",
16+
"intervalFactor": 1,
17+
"interval": "1h",
18+
"database": "default",
19+
"table": "test_lttb",
20+
"maxDataPoints": 569,
21+
"timeRange": {
22+
"from": "2025-03-02T22:18:06.471Z",
23+
"to": "2025-04-01T22:18:06.471Z"
24+
}
25+
}

‎src/datasource/datasource.ts

+5-3
Original file line numberDiff line numberDiff line change
@@ -780,8 +780,8 @@ export class CHDataSource
780780
to: options.range.to.toISOString(), // Convert to Unix timestamp
781781
},
782782
};
783-
const createQueryResult = await this.wasmModule.createQuery(queryData);
784-
const { sql, keys, error } = createQueryResult
783+
const createQueryResult = await this.wasmModule.createQuery(queryData);
784+
let { sql, error } = createQueryResult
785785

786786
if (error) {
787787
throw new Error(error);
@@ -814,8 +814,10 @@ export class CHDataSource
814814
scopedVars,
815815
interpolateQueryExpr
816816
);
817+
818+
const { properties } = await this.wasmModule.getAstProperty(interpolatedQuery, 'group by')
817819

818-
return { stmt: interpolatedQuery, keys: keys };
820+
return { stmt: interpolatedQuery, keys: properties };
819821
} catch (error) {
820822
// Propagate the error instead of returning a default value
821823
throw error;

‎src/datasource/sql-series/toTimeSeries.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ export const toTimeSeries = (extrapolate = true, self): any => {
159159
});
160160

161161
each(metrics, function (dataPoints, seriesName) {
162-
const processedDataPoints = extrapolate ? extrapolateDataPoints(dataPoints, self) : dataPoints;
162+
const processedDataPoints = (extrapolate ? extrapolateDataPoints(dataPoints, self) : dataPoints).filter(item => (typeof item[0] === 'number' || item[0] === null) && item[1]);
163163

164164
timeSeries.push({
165165
length: processedDataPoints.length,

0 commit comments

Comments
 (0)