[Grafana] Handle format parameter in date_histogram (#1322)

Author: trzysiek

platform/model/bucket_aggregations/date_histogram.go

@@ -34,6 +34,7 @@ type DateHistogram struct {
 	field             model.Expr // name of the field, e.g. timestamp
 	interval          string
 	timezone          string
+	defaultFormat     bool           // how we format "key_as_string". If not default, it's milliseconds
 	wantedTimezone    *time.Location // key is in `timezone` time, and we need it to be UTC
 	extendedBoundsMin int64
 	extendedBoundsMax int64
@@ -42,7 +43,7 @@ type DateHistogram struct {
 	fieldDateTimeType clickhouse.DateTimeType
 }
 
-func NewDateHistogram(ctx context.Context, field model.Expr, interval, timezone string, minDocCount int,
+func NewDateHistogram(ctx context.Context, field model.Expr, interval, timezone, format string, minDocCount int,
 	extendedBoundsMin, extendedBoundsMax int64, intervalType DateHistogramIntervalType, fieldDateTimeType clickhouse.DateTimeType) *DateHistogram {
 
 	wantedTimezone, err := time.LoadLocation(timezone)
@@ -51,9 +52,11 @@ func NewDateHistogram(ctx context.Context, field model.Expr, interval, timezone
 		wantedTimezone = time.UTC
 	}
 
+	defaultFormat := format != "epoch_millis"
+
 	return &DateHistogram{ctx: ctx, field: field, interval: interval, timezone: timezone, wantedTimezone: wantedTimezone,
 		minDocCount: minDocCount, extendedBoundsMin: extendedBoundsMin, extendedBoundsMax: extendedBoundsMax,
-		intervalType: intervalType, fieldDateTimeType: fieldDateTimeType}
+		intervalType: intervalType, fieldDateTimeType: fieldDateTimeType, defaultFormat: defaultFormat}
 }
 
 func (typ DateHistogramIntervalType) String(ctx context.Context) string {
@@ -100,12 +103,17 @@ func (query *DateHistogram) TranslateSqlResponseToJson(rows []model.QueryResultR
 		}
 		originalKey := query.getKey(row)
 		responseKey := query.calculateResponseKey(originalKey)
-
+		var keyAsString string
+		if query.defaultFormat {
+			keyAsString = query.calculateKeyAsString(responseKey)
+		} else {
+			keyAsString = strconv.FormatInt(responseKey, 10)
+		}
 		response = append(response, model.JsonMap{
 			OriginalKeyName: originalKey,
 			"key":           responseKey,
 			"doc_count":     docCount,
-			"key_as_string": query.calculateKeyAsString(responseKey),
+			"key_as_string": keyAsString,
 		})
 	}
 

platform/model/bucket_aggregations/date_histogram_test.go

@@ -22,6 +22,6 @@ func TestTranslateSqlResponseToJson(t *testing.T) {
 		},
 	}
 	response := (&DateHistogram{interval: interval, extendedBoundsMax: NoExtendedBound, extendedBoundsMin: NoExtendedBound,
-		intervalType: DateHistogramFixedInterval, wantedTimezone: time.UTC}).TranslateSqlResponseToJson(resultRows)
+		defaultFormat: true, intervalType: DateHistogramFixedInterval, wantedTimezone: time.UTC}).TranslateSqlResponseToJson(resultRows)
 	assert.Equal(t, expectedResponse, response)
 }

platform/parsers/elastic_query_dsl/aggregation_parser_test.go

@@ -658,6 +658,7 @@ func allAggregationTests() []testdata.AggregationTestCase {
 	add(testdata.AggregationTests, "agg_req")
 	add(testdata.AggregationTests2, "agg_req_2")
 	add(testdata.AggregationTestsWithDates, "dates")
+	add(testdata.GrafanaAggregationTests, "grafana")
 	add(testdata.KibanaSampleDataLogs, "kibana-sample-data-logs")
 	add(testdata.PipelineAggregationTests, "pipeline_agg_req")
 	add(dashboard_1.AggregationTests, "dashboard-1/agg_req")

platform/parsers/elastic_query_dsl/pancake_aggregation_parser_buckets.go

@@ -135,6 +135,7 @@ func (cw *ClickhouseQueryTranslator) parseDateHistogram(aggregation *pancakeAggr
 
 	minDocCount := cw.parseMinDocCount(params)
 	timezone := cw.parseStringField(params, "time_zone", "")
+	format := cw.parseStringField(params, "format", "")
 	interval, intervalType := cw.extractInterval(params)
 	// TODO  GetDateTimeTypeFromExpr can be moved and it should take cw.Schema as an argument
 
@@ -143,7 +144,7 @@ func (cw *ClickhouseQueryTranslator) parseDateHistogram(aggregation *pancakeAggr
 	}
 
 	dateHistogram := bucket_aggregations.NewDateHistogram(cw.Ctx,
-		field, interval, timezone, minDocCount, ebMin, ebMax, intervalType, dateTimeType)
+		field, interval, timezone, format, minDocCount, ebMin, ebMax, intervalType, dateTimeType)
 	aggregation.queryType = dateHistogram
 
 	columnSql := dateHistogram.GenerateSQL()

platform/testdata/grafana.go

@@ -0,0 +1,86 @@
+// Copyright Quesma, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+package testdata
+
+import "github.com/QuesmaOrg/quesma/platform/model"
+
+var GrafanaAggregationTests = []AggregationTestCase{
+	{ // [0]
+		TestName: "simple max/min aggregation as 2 siblings",
+		QueryRequestJson: `
+		{
+			"aggs": {
+				"2": {
+					"date_histogram": {
+						"field": "@timestamp",
+						"fixed_interval": "2000ms",
+						"min_doc_count": 0,
+						"extended_bounds": {
+							"min": 1740930494000,
+							"max": 1740930500000
+						},
+						"format": "epoch_millis"
+					}
+				}
+			},
+			"size": 0
+		}`,
+		ExpectedResponse: `
+		{
+			"aggregations": {
+                "2": {
+                    "buckets": [
+                        {
+                            "doc_count": 0,
+                            "key": 1740930494000,
+                            "key_as_string": "1740930494000"
+                        },
+                        {
+                            "doc_count": 4,
+                            "key": 1740930496000,
+                            "key_as_string": "1740930496000"
+                        },
+                        {
+                            "doc_count": 0,
+                            "key": 1740930498000,
+                            "key_as_string": "1740930498000"
+                        },
+                        {
+                            "doc_count": 1,
+                            "key": 1740930500000,
+                            "key_as_string": "1740930500000"
+                        }
+                    ]
+                }
+            },
+            "hits": {
+                "hits": [],
+                "max_score": null,
+                "total": {
+                    "relation": "eq",
+                    "value": 7
+                }
+            },
+            "status": 200,
+            "timed_out": false,
+            "took": 30
+        }`,
+		ExpectedPancakeResults: []model.QueryResultRow{
+			{Cols: []model.QueryResultCol{
+				model.NewQueryResultCol("aggr__2__key_0", int64(1740930496000/2000)),
+				model.NewQueryResultCol("aggr__2__count", int64(4)),
+			}},
+			{Cols: []model.QueryResultCol{
+				model.NewQueryResultCol("aggr__2__key_0", int64(1740930500000/2000)),
+				model.NewQueryResultCol("aggr__2__count", int64(1)),
+			}},
+		},
+		ExpectedPancakeSQL: `
+			SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 2000) AS "aggr__2__key_0",
+			  count(*) AS "aggr__2__count"
+			FROM __quesma_table_name
+			GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 2000) AS
+			  "aggr__2__key_0"
+			ORDER BY "aggr__2__key_0" ASC`,
+	},
+}