diff --git a/pkg/unify-query/Makefile b/pkg/unify-query/Makefile index 9e1723ec5..f5b1c5adc 100644 --- a/pkg/unify-query/Makefile +++ b/pkg/unify-query/Makefile @@ -50,3 +50,7 @@ addlicense: .PHONY: imports imports: addlicense goimports-reviser -rm-unused -set-alias -format -project-name "github.com/TencentBlueKing/bkmonitor-datalink/pkg" ./... + +.PHONY: race +race: + go test -race -timeout 1m -parallel 8 ./... \ No newline at end of file diff --git a/pkg/unify-query/influxdb/router_mock.go b/pkg/unify-query/influxdb/router_mock.go index 3ac53218b..34979c290 100644 --- a/pkg/unify-query/influxdb/router_mock.go +++ b/pkg/unify-query/influxdb/router_mock.go @@ -34,6 +34,7 @@ const ( ResultTableEs = "result_table.es" ResultTableBkBaseEs = "result_table.bk_base_es" ResultTableBkSQL = "result_table.bk_sql" + ResultTableDoris = "result_table.doris" ) var ( @@ -123,7 +124,7 @@ func MockSpaceRouter(ctx context.Context) { ) tsdb.SetStorage("2", &tsdb.Storage{Type: consul.InfluxDBStorageType}) tsdb.SetStorage("3", &tsdb.Storage{Type: consul.ElasticsearchStorageType, Address: mock.EsUrl}) - tsdb.SetStorage("4", &tsdb.Storage{Type: consul.BkSqlStorageType, Address: mock.BkSQLUrl}) + tsdb.SetStorage("4", &tsdb.Storage{Type: consul.BkSqlStorageType, Address: mock.BkBaseUrl}) r := GetInfluxDBRouter() r.clusterInfo = ir.ClusterInfo{ @@ -183,6 +184,9 @@ func MockSpaceRouter(ctx context.Context) { ResultTableBkBaseEs: &ir.SpaceResultTable{ TableId: ResultTableBkBaseEs, }, + ResultTableDoris: &ir.SpaceResultTable{ + TableId: ResultTableDoris, + }, }, }, ir.ResultTableDetailInfo{ @@ -195,6 +199,7 @@ func MockSpaceRouter(ctx context.Context) { Measurement: "kubelet_info", BcsClusterID: "BCS-K8S-00000", MeasurementType: redis.BkSplitMeasurement, + StorageType: consul.VictoriaMetricsStorageType, DataLabel: "kubelet_info", }, "system.cpu_summary": &ir.ResultTableDetail{ @@ -206,6 +211,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "cpu_summary", }, "system.cpu_detail": &ir.ResultTableDetail{ @@ -214,6 +220,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "100147_ieod_system_cpu_detail_raw", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "cpu_detail", }, "system.disk": &ir.ResultTableDetail{ @@ -222,6 +229,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "100147_ieod_system_disk_raw", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "disk", }, ResultTableVM: &ir.ResultTableDetail{ @@ -231,6 +239,7 @@ func MockSpaceRouter(ctx context.Context) { Fields: vmFiedls, BcsClusterID: "BCS-K8S-00000", MeasurementType: redis.BkSplitMeasurement, + StorageType: consul.VictoriaMetricsStorageType, DataLabel: "vm", }, ResultTableInfluxDB: &ir.ResultTableDetail{ @@ -243,12 +252,14 @@ func MockSpaceRouter(ctx context.Context) { MeasurementType: redis.BkSplitMeasurement, ClusterName: "default", DataLabel: "influxdb", + StorageType: consul.InfluxDBStorageType, }, ResultTableEs: &ir.ResultTableDetail{ - StorageId: 3, - TableId: ResultTableEs, - DB: "es_index", - SourceType: "bkdata", + StorageId: 3, + TableId: ResultTableEs, + DB: "es_index", + SourceType: "", + StorageType: consul.ElasticsearchStorageType, StorageClusterRecords: []ir.Record{ { StorageID: 3, @@ -264,14 +275,25 @@ func MockSpaceRouter(ctx context.Context) { DataLabel: "es", }, ResultTableBkSQL: &ir.ResultTableDetail{ - StorageId: 4, - TableId: ResultTableBkSQL, - DataLabel: "bksql", + StorageId: 4, + TableId: ResultTableBkSQL, + DataLabel: "bksql", + DB: "2_bklog_bkunify_query_doris", + StorageType: consul.BkSqlStorageType, + }, + ResultTableDoris: &ir.ResultTableDetail{ + StorageId: 4, + TableId: ResultTableDoris, + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + DataLabel: "bksql", + StorageType: consul.BkSqlStorageType, }, ResultTableBkBaseEs: &ir.ResultTableDetail{ - SourceType: "bkdata", - DB: "es_index", - DataLabel: "bkbase_es", + SourceType: "bkdata", + DB: "es_index", + DataLabel: "bkbase_es", + StorageType: consul.ElasticsearchStorageType, }, }, nil, ir.DataLabelToResultTable{ diff --git a/pkg/unify-query/influxdb/router_test.go b/pkg/unify-query/influxdb/router_test.go deleted file mode 100644 index 28b896fec..000000000 --- a/pkg/unify-query/influxdb/router_test.go +++ /dev/null @@ -1,477 +0,0 @@ -// Tencent is pleased to support the open source community by making -// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. -// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. -// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://opensource.org/licenses/MIT -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -package influxdb - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strings" - "sync" - "testing" - "time" - - "github.com/hashicorp/consul/api" - "github.com/influxdata/influxql" - "github.com/prashantv/gostub" - "github.com/stretchr/testify/assert" - - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" -) - -// FakeConsulRouter -func FakeConsulRouter(t *testing.T) *gostub.Stubs { - log.InitTestLogger() - _ = consul.SetInstance( - context.Background(), "", "test-unify", "http://127.0.0.1:8500", - []string{}, "127.0.0.1", 10205, "30s", "", "", "", - ) - consul.MetadataPath = "test/metadata/v1/default/data_id" - consul.MetricRouterPath = "test/metadata/influxdb_metrics" - - res := map[string]api.KVPairs{ - // dataid metadata - consul.MetadataPath: { - { - Key: consul.MetadataPath + "/1500009", - Value: []byte(`{"bk_data_id":1500009,"data_id":1500009,"mq_config":{"storage_config":{"topic":"0bkmonitor_15000090","partition":1},"cluster_config":{"domain_name":"kafka.service.consul","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","result_table_list":[{"bk_biz_id":2,"result_table":"process.port","shipper_list":[{"storage_config":{"real_table_name":"port","database":"process","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[{"field_name":"alive","type":"float","tag":"metric","default_value":"0","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_biz_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_collect_config_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_cloud_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_ip","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_service_category_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_topo_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_topo_level","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"listen_address","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"listen_port","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"pid","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"process_name","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"target","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"time","type":"timestamp","tag":"timestamp","default_value":"","is_config_by_user":true,"description":"\u6570\u636e\u4e0a\u62a5\u65f6\u95f4","unit":"","alias_name":"","option":{}}],"schema_type":"free","option":{}}],"option":{"inject_local_time":true,"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1500009/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"custom","token":"4774c8313d74430ca68c204aa6491eee","transfer_cluster_id":"default"}`), - }, - { - Key: consul.MetadataPath + "/1500015", - Value: []byte(`{"bk_biz_id":2,"bk_data_id":1500015,"data_id":1500015,"mq_config":{"storage_config":{"topic":"0bkmonitor_15000150","partition":1},"cluster_config":{"domain_name":"kafka.service.consul","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_event","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_event_public_1500015","shipper_list":[{"storage_config":{"index_datetime_format":"write_20060102","index_datetime_timezone":0,"date_format":"%Y%m%d","slice_size":500,"slice_gap":1440,"retention":30,"warm_phase_days":0,"warm_phase_settings":{},"base_index":"2_bkmonitor_event_public_1500015","index_settings":{"number_of_shards":4,"number_of_replicas":1},"mapping_settings":{"dynamic_templates":[{"discover_dimension":{"path_match":"dimensions.*","mapping":{"type":"keyword"}}}]}},"cluster_config":{"domain_name":"es7.service.consul","port":9200,"schema":null,"is_ssl_verify":false,"cluster_id":3,"cluster_name":"es7_cluster","version":"7.2","custom_option":"","registered_system":"_default","creator":"system","create_time":1624001652,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"elasticsearch","auth_info":{"password":"5gYTZqvd7Z7s","username":"elastic"}}],"field_list":[{"field_name":"dimensions","type":"object","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"object","es_dynamic":true}},{"field_name":"event","type":"object","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"object","es_properties":{"content":{"type":"text"},"count":{"type":"integer"}}}},{"field_name":"event_name","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"keyword"}},{"field_name":"target","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"keyword"}},{"field_name":"time","type":"timestamp","tag":"timestamp","default_value":"","is_config_by_user":true,"description":"\u6570\u636e\u4e0a\u62a5\u65f6\u95f4","unit":"","alias_name":"","option":{"es_type":"date_nanos","es_format":"epoch_millis"}}],"schema_type":"free","option":{"es_unique_field_list":["event","target","dimensions","event_name","time"]}}],"option":{"inject_local_time":true,"timestamp_precision":"ms","flat_batch_key":"data"},"type_label":"log","source_label":"bk_monitor","token":"d6dc05057e384f6db70e3542e3f8a2ce","transfer_cluster_id":"default"}`), - }, - { - Key: consul.MetadataPath + "/1573194", - Value: []byte(`{"bk_data_id":1573194,"data_id":1573194,"mq_config":{"storage_config":{"topic":"0bkmonitor_15731940","partition":7},"cluster_config":{"domain_name":"127.0.0.1","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","option":{"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1573194/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"bk_monitor","token":"9e180be8199946d3a8645639e236a50c","transfer_cluster_id":"default","data_name":"bcs_BCS-K8S-00000_k8s_metric","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_time_series_1573194.__default__","shipper_list":[{"storage_config":{"real_table_name":"__default__","database":"2_bkmonitor_time_series_1573194","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[],"schema_type":"free","option":{"is_split_measurement":true}}]}`), - }, - { - Key: consul.MetadataPath + "/1573195", - Value: []byte(`{"bk_data_id":1573195,"data_id":1573195,"mq_config":{"storage_config":{"topic":"0bkmonitor_15731950","partition":6},"cluster_config":{"domain_name":"127.0.0.1","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","option":{"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1573195/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"bk_monitor","token":"f383f2f00f0449e7ba849fc8b6e28bad","transfer_cluster_id":"default","data_name":"bcs_BCS-K8S-00000_custom_metric","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_time_series_1573195.__default__","shipper_list":[{"storage_config":{"real_table_name":"__default__","database":"2_bkmonitor_time_series_1573195","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[],"schema_type":"free","option":{"is_split_measurement":true}}]}`), - }, - }, - // metric info - consul.MetricRouterPath: { - { - Key: consul.MetricRouterPath + "/1500009/time_series_metric", - Value: []byte(`["alive"]`), - }, - { - Key: consul.MetricRouterPath + "/1573194/time_series_metric", - Value: []byte(`["node_memory_Cached_bytes", "node_filesystem_files"]`), - }, - { - Key: consul.MetricRouterPath + "/1573195/time_series_metric", - Value: []byte(`["node_memory_Cached_bytes", "node_filesystem_files"]`), - }, - }, - } - - consul.GetDataWithPrefix = func(p string) (api.KVPairs, error) { - return res[p], nil - } - consul.GetPathDataIDPath = func(metadataPath, version string) ([]string, error) { - return []string{metadataPath}, nil - } - stubs := gostub.New() - - _ = consul.ReloadBCSInfo() - reloadData, err := consul.ReloadRouterInfo() - assert.Nil(t, err) - ReloadTableInfos(reloadData) - metricData, err := consul.ReloadMetricInfo() - assert.Nil(t, err) - ReloadMetricRouter(metricData) - - return stubs -} - -// TestRouter_DBBiz -func TestRouter_DBBiz(t *testing.T) { - stubs := FakeConsulRouter(t) - defer stubs.Reset() - - expectsTsDBRouter := NewTsDBRouter() - expectsTsDBRouter.AddTables(1500009, []*consul.TableID{ - {DB: "process", Measurement: "port", IsSplitMeasurement: false, ClusterID: "2"}, - }) - expectsTsDBRouter.AddTables(1573194, []*consul.TableID{ - {DB: "2_bkmonitor_time_series_1573194", Measurement: "", IsSplitMeasurement: true, ClusterID: "2"}, - }) - expectsTsDBRouter.AddTables(1573195, []*consul.TableID{ - {DB: "2_bkmonitor_time_series_1573195", Measurement: "", IsSplitMeasurement: true, ClusterID: "2"}, - }) - - expectBizRouter := NewBizRouter() - expectBizRouter.AddRouter(2, []consul.DataID{ - 1500009, 1500015, 1573194, 1573195, - }...) - - expectMetricMap := NewMetricRouter() - expectMetricMap.AddRouter("alive", []consul.DataID{1500009}...) - expectMetricMap.AddRouter("node_memory_Cached_bytes", []consul.DataID{1573194, 1573195}...) - expectMetricMap.AddRouter("node_filesystem_files", []consul.DataID{1573194, 1573195}...) - - var dataIDs = []consul.DataID{1500009, 1500015, 1573194, 1573195} - var bizID = 2 - var metrics = []string{"alive", "node_memory_Cached_bytes", "node_filesystem_files"} - - assert.Equal(t, - expectsTsDBRouter.GetTableIDs(dataIDs...), - tsDBRouter.GetTableIDs(dataIDs...), - ) - assert.Equal(t, - expectBizRouter.GetRouter(bizID), - bizRouter.GetRouter(bizID), - ) - - assert.Equal(t, - expectMetricMap.GetRouter(metrics...), - metricRouter.GetRouter(metrics...), - ) -} - -const ( - ReferenceName = "m" - COUNT = "count" -) - -// ctxKeyPromQueryKeyName -type ctxKeyPromQueryKeyName struct { -} - -// ctxKeyPromMetricMappingKey -type ctxKeyPromMetricMappingKey struct { -} - -// QueryInfo -type QueryInfo struct { - DB string - Measurement string - DataIDList []consul.DataID - - // 是否为行转列表 - IsPivotTable bool -} - -// QueryInfoIntoContext -func QueryInfoIntoContext(ctx context.Context, metrics string, queryInfo *QueryInfo) context.Context { - var ( - buffer map[string]*QueryInfo - ok bool - ) - if buffer, ok = ctx.Value(&ctxKeyPromQueryKeyName{}).(map[string]*QueryInfo); !ok { - buffer = make(map[string]*QueryInfo) - } - buffer[metrics] = queryInfo - - ctx = context.WithValue(ctx, &ctxKeyPromQueryKeyName{}, buffer) - return ctx -} - -// MetricMappingIntoContext -func MetricMappingIntoContext(ctx context.Context, referenceName string, metricName string) context.Context { - var mapping map[string]string - var ok bool - if mapping, ok = ctx.Value(&ctxKeyPromMetricMappingKey{}).(map[string]string); !ok { - mapping = make(map[string]string) - } - mapping[referenceName] = metricName - return context.WithValue(ctx, &ctxKeyPromMetricMappingKey{}, mapping) -} - -// MetricMappingFromContext -func MetricMappingFromContext(ctx context.Context, referenceName string) (string, error) { - if mapping, ok := ctx.Value(&ctxKeyPromMetricMappingKey{}).(map[string]string); ok { - if metricName, ok := mapping[referenceName]; ok { - return metricName, nil - } - return "", errors.New("get metric mapping failed") - } - return "", errors.New("get metric mapping failed") -} - -// where类型说明，决定了where语句的渲染格式 -type ValueType int - -const ( - StringType1 ValueType = 0 - NumType ValueType = 1 - RegexpType ValueType = 2 - TextType ValueType = 3 -) - -// 操作符号映射 -type Operator string - -const ( - EqualOperator string = "=" - NEqualOperator string = "!=" - UpperOperator string = ">" - UpperEqualOperator string = ">=" - LowerOperator string = "<" - LowerEqualOperator string = "<=" - RegexpOperator string = "=~" - NRegexpOperator string = "!~" -) - -const ( - AndOperator string = "and" - OrOperator string = "or" -) - -// WhereList -type WhereList struct { - whereList []*Where - logicalList []string -} - -// NewWhereList -func NewWhereList() *WhereList { - return &WhereList{ - whereList: make([]*Where, 0), - logicalList: make([]string, 0), - } -} - -// Append -func (l *WhereList) Append(logicalOperator string, where *Where) { - l.logicalList = append(l.logicalList, logicalOperator) - l.whereList = append(l.whereList, where) -} - -// String -func (l *WhereList) String() string { - b := new(strings.Builder) - for index, where := range l.whereList { - if index == 0 { - b.WriteString("where ") - } else { - b.WriteString(" " + l.logicalList[index-1] + " ") - } - b.WriteString(where.String()) - } - return b.String() -} - -// Check 判断条件里是包含tag的值，例如：tagName: bk_biz_id，tagValue：[1, 2]，bk_biz_id = 1 和 bk_biz_id = 2 都符合 -func (l *WhereList) Check(tagName string, tagValue []string) bool { - tagMap := make(map[string]interface{}) - for _, v := range tagValue { - tagMap[v] = nil - } - for _, w := range l.whereList { - if w.Name == tagName && w.ValueType == StringType1 && w.Operator == EqualOperator { - if _, ok := tagMap[w.Value]; ok { - return true - } - } - } - return false -} - -// Where -type Where struct { - Name string - Value string - Operator string - ValueType ValueType -} - -// String -func (w *Where) String() string { - switch w.ValueType { - case NumType: - return fmt.Sprintf("%s %s %s", w.Name, w.Operator, w.Value) - case RegexpType: - // influxdb 中以 "/" 为分隔符，所以这里将正则中的 "/" 做个简单的转义 "\/" - return fmt.Sprintf("%s %s /%s/", w.Name, w.Operator, strings.ReplaceAll(w.Value, "/", "\\/")) - case TextType: - // 直接将长文本追加，作为一种特殊处理逻辑，influxQL 需要转义反斜杠 - return fmt.Sprintf("%s", strings.ReplaceAll(w.Value, "\\", "\\\\")) - default: - // 默认为字符串类型 - return fmt.Sprintf("%s %s '%s'", w.Name, w.Operator, w.Value) - } -} - -// 这里降低influxdb流量，主要不是根据时间减少点数，而是预先聚合减少series数量 -func generateSQL(field, measurement, db, aggregation string, whereList *WhereList, sLimit, limit int, dimensions []string, window time.Duration) (string, bool, bool) { - - var ( - groupingStr string - isWithGroupBy bool - withTag = ",*::tag" - limitStr string - sLimitStr string - aggField string - rpName string - isCountGroup bool - newAggregation string - ) - - newAggregation = aggregation - //rpName, field, newAggregation = GetRp(db, measurement, field, aggregation, window, whereList) - - // 根据RP重新生成measurement - if rpName != "" { - measurement = fmt.Sprintf("\"%s\".\"%s\"", rpName, measurement) - } else { - measurement = fmt.Sprintf("\"%s\"", measurement) - } - - // 存在聚合条件，需要增加聚合 - if newAggregation != "" && window != 0 { - var groupList []string - isWithGroupBy = true - isCountGroup = aggregation == COUNT - - if len(dimensions) > 0 { - for _, d := range dimensions { - groupList = append(groupList, fmt.Sprintf("\"%s\"", d)) - } - } - groupList = append(groupList, "time("+window.String()+")") - groupingStr = " group by " + strings.Join(groupList, ",") - withTag = "" - aggField = fmt.Sprintf("%s(\"%s\")", newAggregation, field) - // 由于此处存在聚合，所以可以直接使用influxdb的limit能力 - if sLimit > 0 { - sLimitStr = fmt.Sprintf(" slimit %d", sLimit) - } - } else { - aggField = fmt.Sprintf("\"%s\"", field) - } - - if limit > 0 { - limitStr = fmt.Sprintf(" limit %d", limit) - } - - return fmt.Sprintf("select %s as %s,time as %s%s from %s %s%s%s%s", - aggField, ResultColumnName, TimeColumnName, withTag, measurement, whereList.String(), groupingStr, limitStr, sLimitStr, - ), isWithGroupBy, isCountGroup -} - -// do -func do(ctx context.Context) []string { - metric, err := MetricMappingFromContext(ctx, ReferenceName) - if err != nil { - panic(err) - } - dataIDs := NewDataIDFilter(metric).FilterByBizIDs(2).FilterByProjectIDs(). - FilterByClusterIDs().Values() - - tableList := make([]*consul.TableID, 0) - for _, dataID := range dataIDs { - tableInfo := GetTableIDsByDataID(dataID) - if len(tableInfo) == 0 { - continue - } - tableList = append(tableList, tableInfo...) - } - - var res []string - where := NewWhereList() - for _, t := range tableList { - sql, _, _ := generateSQL("value", metric, t.DB, "max", where, 100, 100, nil, 5*time.Minute) - res = append(res, sql) - } - return res -} - -// TestGetTableIDByDBAndMeasurement -func TestGetTableIDByDBAndMeasurement(t *testing.T) { - stubs := FakeConsulRouter(t) - defer stubs.Reset() - - metricName := []string{"node_memory_Cached_bytes", "node_filesystem_files"} - - queryInfo := QueryInfo{ - DB: "", - Measurement: "", - DataIDList: nil, - IsPivotTable: false, - } - - var wg sync.WaitGroup - var wgCh sync.WaitGroup - var num = 1000 - - var checkSQL = map[string]string{ - "node_filesystem_files": "select max(\"value\") as _value,time as _time from \"node_filesystem_files\" group by time(5m0s) limit 100 slimit 100", - "node_memory_Cached_bytes": "select max(\"value\") as _value,time as _time from \"node_memory_Cached_bytes\" group by time(5m0s) limit 100 slimit 100", - } - var ch = map[string]chan []string{ - "node_filesystem_files": make(chan []string, 0), - "node_memory_Cached_bytes": make(chan []string, 0), - } - - wg.Add(1) - go func() { - defer wg.Done() - num = rand.Intn(50) - for i := 0; i < num; i++ { - go func() { - time.Sleep(time.Duration(num) * time.Millisecond) - data, err := consul.ReloadRouterInfo() - assert.Nil(t, err) - ReloadTableInfos(data) - - metricData, err := consul.ReloadMetricInfo() - assert.Nil(t, err) - ReloadMetricRouter(metricData) - }() - } - fmt.Printf("fresh %d\n", num) - time.Sleep(time.Second) - }() - - for _, metric := range metricName { - wgCh.Add(1) - go func(metric string) { - defer wgCh.Done() - for c := range ch[metric] { - for _, sql := range c { - _, err := influxql.ParseQuery(sql) - assert.Equal(t, sql, checkSQL[metric]) - assert.Nil(t, err) - t.Log(sql) - } - } - }(metric) - - ctx := context.Background() - ctx = QueryInfoIntoContext(ctx, ReferenceName, &queryInfo) - ctx = MetricMappingIntoContext(ctx, ReferenceName, metric) - - var ticker = time.NewTicker(5 * time.Second) - go func(metric string) { - defer ticker.Stop() - for { - for i := 0; i < num; i++ { - wg.Add(1) - go func(metric string) { - defer wg.Done() - ch[metric] <- do(ctx) - }(metric) - } - select { - case <-ticker.C: - close(ch[metric]) - return - } - } - }(metric) - } - wg.Wait() - fmt.Println(ch) - wgCh.Wait() -} diff --git a/pkg/unify-query/internal/json/custom.go b/pkg/unify-query/internal/json/custom.go index 01979ae13..675a4b19d 100644 --- a/pkg/unify-query/internal/json/custom.go +++ b/pkg/unify-query/internal/json/custom.go @@ -10,11 +10,44 @@ package json import ( + "encoding/json" "fmt" "sort" "strings" ) +const ( + StepString = "." +) + +func mapData(prefix string, data map[string]any, res map[string]any) { + for k, v := range data { + if prefix != "" { + k = prefix + StepString + k + } + switch v.(type) { + case map[string]any: + mapData(k, v.(map[string]any), res) + default: + res[k] = v + } + } +} + +// ParseObject 解析 json，按照层级打平 +func ParseObject(prefix, intput string) (map[string]any, error) { + oldData := make(map[string]any) + newData := make(map[string]any) + + err := json.Unmarshal([]byte(intput), &oldData) + if err != nil { + return newData, err + } + + mapData(prefix, oldData, newData) + return newData, nil +} + func MarshalListMap(data []map[string]interface{}) string { if len(data) == 0 { return "[]" diff --git a/pkg/unify-query/internal/json/custom_test.go b/pkg/unify-query/internal/json/custom_test.go new file mode 100644 index 000000000..8e1584ead --- /dev/null +++ b/pkg/unify-query/internal/json/custom_test.go @@ -0,0 +1,112 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package json_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" +) + +func TestParseJson(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]any + wantErr bool + }{ + { + name: "test1", + input: `{"a": "b"}`, + expected: map[string]any{ + "__ext.a": "b", + }, + }, + { + name: "normal nested json", + input: `{"a": {"b": 1, "c": "test"}, "d": true}`, + expected: map[string]interface{}{ + "__ext.a.b": float64(1), + "__ext.a.c": "test", + "__ext.d": true, + }, + wantErr: false, + }, + { + name: "single level json", + input: `{"key1": "value1", "key2": 123}`, + expected: map[string]interface{}{ + "__ext.key1": "value1", + "__ext.key2": float64(123), // JSON numbers are decoded as float64 + }, + wantErr: false, + }, + { + name: "empty json", + input: `{}`, + expected: map[string]interface{}{}, + wantErr: false, + }, + { + name: "invalid json", + input: `{"key": "value"`, + expected: map[string]interface{}{}, + wantErr: true, + }, + { + name: "json with special characters in keys", + input: `{"a.b": {"c-d": "value"}}`, + expected: map[string]interface{}{ + "__ext.a.b.c-d": "value", + }, + wantErr: false, + }, + { + name: "deeply nested json", + input: `{"a": {"b": {"c": {"d": "value"}}}}`, + expected: map[string]interface{}{ + "__ext.a.b.c.d": "value", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := json.ParseObject("__ext", tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseObject() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + assert.Equal(t, got, tt.expected) + } + }) + } +} + +func TestParseJson_WithArrays(t *testing.T) { + // 虽然当前实现不支持数组，但测试一下看看行为 + input := `{"a": [1, 2, 3], "b": {"c": [4, 5]}}` + expected := map[string]any{ + "__ext.a": []any{float64(1), float64(2), float64(3)}, + "__ext.b.c": []any{float64(4), float64(5)}, + } + got, err := json.ParseObject("__ext", input) + if err != nil { + t.Errorf("ParseObject() with arrays returned error: %v", err) + return + } + + assert.Equal(t, got, expected) +} diff --git a/pkg/unify-query/internal/querystring/expr.go b/pkg/unify-query/internal/querystring/expr.go new file mode 100644 index 000000000..c070dbbc7 --- /dev/null +++ b/pkg/unify-query/internal/querystring/expr.go @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +import ( + "strings" + "time" +) + +// Expr . +type Expr interface { +} + +// AndExpr . +type AndExpr struct { + Left Expr + Right Expr +} + +// NewAndExpr . +func NewAndExpr(left, right Expr) *AndExpr { + return &AndExpr{Left: left, Right: right} +} + +// OrExpr . +type OrExpr struct { + Left Expr + Right Expr +} + +// NewOrExpr . +func NewOrExpr(left, right Expr) *OrExpr { + return &OrExpr{Left: left, Right: right} +} + +// NotExpr . +type NotExpr struct { + Expr Expr +} + +// NewNotExpr . +func NewNotExpr(q Expr) *NotExpr { + return &NotExpr{Expr: q} +} + +// FieldableExpr . +type FieldableExpr interface { + SetField(field string) +} + +// MatchExpr . +type MatchExpr struct { + Field string + Value string +} + +// NewMatchExpr . +func NewMatchExpr(s string) *MatchExpr { + return &MatchExpr{ + Value: s, + } +} + +// SetField . +func (q *MatchExpr) SetField(field string) { + q.Field = field +} + +// RegexpExpr . +type RegexpExpr struct { + Field string + Value string +} + +// NewRegexpExpr . +func NewRegexpExpr(s string) *RegexpExpr { + return &RegexpExpr{ + Value: s, + } +} + +// SetField . +func (q *RegexpExpr) SetField(field string) { + q.Field = field +} + +// WildcardExpr . +type WildcardExpr struct { + Field string + Value string +} + +// NewWildcardExpr . +func NewWildcardExpr(s string) *WildcardExpr { + return &WildcardExpr{ + Value: s, + } +} + +// SetField . +func (q *WildcardExpr) SetField(field string) { + q.Field = field +} + +// NumberRangeExpr . +type NumberRangeExpr struct { + Field string + Start *string + End *string + IncludeStart bool + IncludeEnd bool +} + +// NewNumberRangeExpr . +func NewNumberRangeExpr(start, end *string, includeStart, includeEnd bool) *NumberRangeExpr { + return &NumberRangeExpr{ + Start: start, + End: end, + IncludeStart: includeStart, + IncludeEnd: includeEnd, + } +} + +// SetField . +func (q *NumberRangeExpr) SetField(field string) { + q.Field = field +} + +// TimeRangeExpr . +type TimeRangeExpr struct { + Field string + Start *string + End *string + IncludeStart bool + IncludeEnd bool +} + +// NewTimeRangeExpr . +func NewTimeRangeExpr(start, end *string, includeStart, includeEnd bool) *TimeRangeExpr { + return &TimeRangeExpr{ + Start: start, + End: end, + IncludeStart: includeStart, + IncludeEnd: includeEnd, + } +} + +// SetField . +func (q *TimeRangeExpr) SetField(field string) { + q.Field = field +} + +func queryTimeFromString(t string) (time.Time, error) { + return time.Parse(time.RFC3339, t) +} + +func newStringExpr(str string) FieldableExpr { + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + return NewRegexpExpr(str[1 : len(str)-1]) + } + + if strings.ContainsAny(str, "*?") { + return NewWildcardExpr(str) + } + + return NewMatchExpr(str) +} + +type TermExpr struct { + Field string + Value string +} diff --git a/pkg/unify-query/internal/querystring/lex.go b/pkg/unify-query/internal/querystring/lex.go new file mode 100644 index 000000000..3f62e1bb1 --- /dev/null +++ b/pkg/unify-query/internal/querystring/lex.go @@ -0,0 +1,317 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package querystring + +import ( + "bufio" + "bytes" + "io" + "strings" + "unicode" +) + +const reservedChars = "+-=&|>', '<', '=', '(', ')', '[', ']', '{', '}', '*': + l.buf += string(next) + return singleCharOpState, true + } + + switch { + case !l.inEscape && next == '\\': + l.inEscape = true + return startState, true + case unicode.IsDigit(next): + l.buf += string(next) + return inNumOrStrState, true + case !unicode.IsSpace(next): + l.buf += string(next) + return inStrState, true + } + + // doesn't look like anything, just eat it and stay here + l.reset() + return startState, true +} + +func inPhraseState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // unterminated phrase eats the phrase + if eof { + l.Error("unterminated quote") + return nil, false + } + + // only a non-escaped " ends the phrase + if !l.inEscape && (next == '"' || next == '/') { + // end phrase + switch next { + case '"': + l.nextTokenType = tPHRASE + case '/': + l.nextTokenType = tSLASH + } + l.nextToken = &yySymType{ + s: l.buf, + } + //log.Debugf(context.TODO(), "PHRASE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inPhraseState, true +} + +func singleCharOpState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + l.nextToken = &yySymType{} + + var msg bytes.Buffer + msg.WriteString(l.buf) + msg.WriteString(" => ") + switch l.buf { + case "+": + l.nextTokenType = tPLUS + msg.WriteString("tPLUS") + case "-": + l.nextTokenType = tMINUS + msg.WriteString("tMINUS") + case ":": + l.nextTokenType = tCOLON + msg.WriteString("tCOLON") + case ">": + l.nextTokenType = tGREATER + msg.WriteString("tGREATER") + case "<": + l.nextTokenType = tLESS + msg.WriteString("tLESS") + case "=": + l.nextTokenType = tEQUAL + msg.WriteString("tEQUAL") + case "(": + l.nextTokenType = tLEFTBRACKET + msg.WriteString("tLEFTBRACKET") + case ")": + l.nextTokenType = tRIGHTBRACKET + msg.WriteString("tRIGHTBRACKET") + case "[": + l.nextTokenType = tLEFTRANGE + msg.WriteString("tLEFTRANGE") + case "]": + l.nextTokenType = tRIGHTRANGE + msg.WriteString("tRIGHTRANGE") + case "{": + l.nextTokenType = tLEFTBRACES + msg.WriteString("tLEFTBRACES") + case "}": + l.nextTokenType = tRIGHTBRACES + msg.WriteString("tRIGHTBRACES") + case "*": + l.nextTokenType = tSTAR + msg.WriteString("tSTAR") + } + //log.Debugf(context.TODO(), msg.String()) + + l.reset() + return startState, false +} + +func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // only a non-escaped space ends the tilde (or eof) + if eof || (!l.inEscape && next == ' ' || next == ':' || next == ')' || next == ']' || next == '}') { + // end number + consumed := true + if !eof && (next == ':' || next == ')' || next == ']' || next == '}') { + consumed = false + } + + l.nextTokenType = tNUMBER + l.nextToken = &yySymType{ + s: l.buf, + } + //log.Debugf(context.TODO(), "NUMBER - '%s'", l.nextToken.s) + l.reset() + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + return inNumOrStrState, true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + // go directly to string, no successfully or unsuccessfully + // escaped string results in a valid number + return inStrState, true + } + + // see where to go + if !l.seenDot && next == '.' { + // stay in this state + l.seenDot = true + l.buf += string(next) + return inNumOrStrState, true + } else if unicode.IsDigit(next) { + l.buf += string(next) + return inNumOrStrState, true + } + + // doesn't look like an number, transition + l.buf += string(next) + return inStrState, true +} + +func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // end on non-escped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == ')' || next == ']')) { + // end string + consumed := true + if !eof && (next == ':' || next == ')' || next == ']') { + consumed = false + } + + switch strings.ToLower(l.buf) { + case "and": + l.nextTokenType = tAND + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "or": + l.nextTokenType = tOR + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "not": + l.nextTokenType = tNOT + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "to": + l.nextTokenType = tTO + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + } + + l.nextTokenType = tSTRING + l.nextToken = &yySymType{ + s: l.buf, + } + //log.Debugf(context.TODO(), "STRING - '%s'", l.nextToken.s) + l.reset() + + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inStrState, true +} diff --git a/pkg/unify-query/internal/querystring/parser.go b/pkg/unify-query/internal/querystring/parser.go new file mode 100644 index 000000000..f2943edf8 --- /dev/null +++ b/pkg/unify-query/internal/querystring/parser.go @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +//go:generate goyacc -o querystring.y.go querystring.y + +import ( + "fmt" + "strings" +) + +// Parse querystring and return Expr +func Parse(query string) (Expr, error) { + if query == "" || query == "*" { + return nil, nil + } + + lex := newLexerWrapper(newExprStringLex(strings.NewReader(query))) + doParse(lex) + + if len(lex.errs) > 0 { + return nil, fmt.Errorf(strings.Join(lex.errs, "\n")) + } + return lex.expr, nil +} + +func doParse(lex *lexerWrapper) { + defer func() { + r := recover() + if r != nil { + lex.errs = append(lex.errs, fmt.Sprintf("parse error: %v", r)) + } + }() + + yyParse(lex) +} + +const ( + queryShould = iota + queryMust + queryMustNot +) + +type lexerWrapper struct { + lex yyLexer + errs []string + expr Expr +} + +func newLexerWrapper(lex yyLexer) *lexerWrapper { + return &lexerWrapper{ + lex: lex, + expr: nil, + } +} + +func (l *lexerWrapper) Lex(lval *yySymType) int { + return l.lex.Lex(lval) +} + +func (l *lexerWrapper) Error(s string) { + l.errs = append(l.errs, s) +} + +func init() { + yyErrorVerbose = true +} diff --git a/pkg/unify-query/internal/querystring/parser_test.go b/pkg/unify-query/internal/querystring/parser_test.go new file mode 100644 index 000000000..045afe09b --- /dev/null +++ b/pkg/unify-query/internal/querystring/parser_test.go @@ -0,0 +1,296 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" +) + +/* +精确匹配(支持AND、OR)： +author:"John Smith" AND age:20 +字段名匹配(*代表通配符)： +status:active +title:(quick brown) +字段名模糊匹配： +vers\*on:(quick brown) +通配符匹配： +qu?ck bro* +正则匹配： +name:/joh?n(ath[oa]n)/ +范围匹配： +count:[1 TO 5] +count:[1 TO 5} +count:[10 TO *] +*/ +func TestParser(t *testing.T) { + testCases := map[string]struct { + q string + e Expr + }{ + "正常查询": { + q: `test`, + e: &MatchExpr{ + Value: `test`, + }, + }, + "负数查询": { + q: `-test`, + e: &NotExpr{ + Expr: &MatchExpr{ + Value: `test`, + }, + }, + }, + "负数查询多条件": { + q: `-test AND good`, + e: &AndExpr{ + Left: &NotExpr{ + Expr: &MatchExpr{ + Value: `test`, + }, + }, + Right: &MatchExpr{ + Value: `good`, + }, + }, + }, + "通配符匹配": { + q: `qu?ck bro*`, + e: &OrExpr{ + Left: &WildcardExpr{ + Value: "qu?ck", + }, + Right: &WildcardExpr{ + Value: "bro*", + }, + }, + }, + "正则匹配": { + q: `name:/joh?n(ath[oa]n)/`, + e: &RegexpExpr{ + Field: "name", + Value: "joh?n(ath[oa]n)", + }, + }, + "范围匹配，左闭右开": { + q: `count:[1 TO 5}`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("1"), + End: pointer("5"), + IncludeStart: true, + IncludeEnd: false, + }, + }, + "范围匹配": { + q: `count:[1 TO 5]`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("1"), + End: pointer("5"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + "范围匹配（无上限）": { + q: `count:[10 TO *]`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("10"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + "字段匹配": { + q: `status:active`, + e: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + "字段匹配 + 括号": { + q: `status:(active)`, + e: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + "多条件组合，括号调整优先级": { + q: `author:"John Smith" AND (age:20 OR status:active)`, + e: &AndExpr{ + Left: &MatchExpr{ + Field: "author", + Value: "John Smith", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Field: "age", + Value: "20", + }, + Right: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + }, + }, + "多条件组合，and 和 or 的优先级": { + q: `(author:"John Smith" AND age:20) OR status:active`, + e: &OrExpr{ + Left: &AndExpr{ + Left: &MatchExpr{ + Field: "author", + Value: "John Smith", + }, + Right: &MatchExpr{ + Field: "age", + Value: "20", + }, + }, + Right: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + }, + "嵌套逻辑表达式": { + q: `a:1 AND (b:2 OR c:3)`, + e: &AndExpr{ + Left: &MatchExpr{ + Field: "a", + Value: "1", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Field: "b", + Value: "2", + }, + Right: &MatchExpr{ + Field: "c", + Value: "3", + }, + }, + }, + }, + "new-1": { + q: `quick brown +fox -news`, + e: &OrExpr{ + Left: &MatchExpr{ + Value: "quick", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Value: "brown", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Value: "fox", + }, + Right: &NotExpr{ + Expr: &MatchExpr{ + Field: "", + Value: "news", + }, + }, + }, + }, + }, + }, + "模糊匹配": { + q: `quick brown fox`, + e: &OrExpr{ + Left: &MatchExpr{ + Value: "quick", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Value: "brown", + }, + Right: &MatchExpr{ + Value: "fox", + }, + }, + }, + }, + "单个条件精确匹配": { + q: `log: "ERROR MSG"`, + e: &MatchExpr{ + Field: "log", + Value: "ERROR MSG", + }, + }, + "match and time range": { + q: "message: test\\ value AND datetime: [\"2020-01-01T00:00:00\" TO \"2020-12-31T00:00:00\"]", + e: &AndExpr{ + Left: &MatchExpr{ + Field: "message", + Value: "test value", + }, + Right: &TimeRangeExpr{ + Field: "datetime", + Start: pointer("2020-01-01T00:00:00"), + End: pointer("2020-12-31T00:00:00"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + }, + "mixed or / and": { + q: "a: 1 OR (b: 2 and c: 4)", + e: &OrExpr{ + Left: &MatchExpr{ + Field: "a", + Value: "1", + }, + Right: &AndExpr{ + Left: &MatchExpr{ + Field: "b", + Value: "2", + }, + Right: &MatchExpr{ + Field: "c", + Value: "4", + }, + }, + }, + }, + "start": { + q: "a: >100", + e: &NumberRangeExpr{ + Field: "a", + Start: pointer("100"), + }, + }, + } + + metadata.InitMetadata() + mock.Init() + + for name, c := range testCases { + t.Run(name, func(t *testing.T) { + expr, err := Parse(c.q) + if err != nil { + t.Errorf("parse return error, %s", err) + return + } + assert.Equal(t, c.e, expr) + }) + } +} + +func pointer(s string) *string { + return &s +} diff --git a/pkg/unify-query/internal/querystring/querystring.y b/pkg/unify-query/internal/querystring/querystring.y new file mode 100644 index 000000000..377925eaa --- /dev/null +++ b/pkg/unify-query/internal/querystring/querystring.y @@ -0,0 +1,270 @@ +%{ +package querystring +%} + +%union { + s string + n int + e Expr +} + +%token tSTRING tPHRASE tNUMBER tSLASH tSTAR +%token tOR tAND tNOT tTO tPLUS tMINUS tCOLON +%token tLEFTBRACKET tRIGHTBRACKET tLEFTRANGE tRIGHTRANGE tLEFTBRACES tRIGHTBRACES +%token tGREATER tLESS tEQUAL + +%type tSTRING +%type tPHRASE +%type tNUMBER +%type tSTAR +%type tSLASH +%type posOrNegNumber +%type searchBase searchLogicParts searchPart searchLogicPart searchLogicSimplePart +%type searchPrefix + +%left tOR +%left tAND +%nonassoc tLEFTBRACKET tRIGHTBRACKET + +%% + +input: +searchLogicParts { + yylex.(*lexerWrapper).expr = $1 +}; + +searchLogicParts: +searchLogicPart searchLogicParts { + $$ = NewOrExpr($1, $2) +} +| +searchLogicPart { + $$ = $1 +} + +searchLogicPart: +searchLogicSimplePart { + $$ = $1 +} +| +searchLogicSimplePart tOR searchLogicPart { + $$ = NewOrExpr($1, $3) +} +| +searchLogicSimplePart tAND searchLogicPart { + $$ = NewAndExpr($1, $3) +}; + +searchLogicSimplePart: +searchPart { + $$ = $1 +} +| +tLEFTBRACKET searchLogicPart tRIGHTBRACKET { + $$ = $2 +} +| +tNOT searchLogicSimplePart { + $$ = NewNotExpr($2) +}; + +searchPart: +searchPrefix searchBase { + switch($1) { + case queryMustNot: + $$ = NewNotExpr($2) + default: + $$ = $2 + } +} +| +searchBase { + $$ = $1 +}; + +searchPrefix: +tPLUS { + $$ = queryMust +} +| +tMINUS { + $$ = queryMustNot +}; + +searchBase: +tSTRING { + $$ = newStringExpr($1) +} +| +tNUMBER { + $$ = NewMatchExpr($1) +} +| +tPHRASE { + phrase := $1 + q := NewMatchExpr(phrase) + $$ = q +} +| +tSLASH{ + phrase := $1 + q := NewRegexpExpr(phrase) + $$ = q +} +| +tSTRING tCOLON tSTRING { + q := newStringExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET { + q := newStringExpr($4) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON posOrNegNumber { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tPHRASE { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tSLASH { + q := NewRegexpExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(&val, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(&val, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(nil, &val, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(nil, &val, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(&phrase, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(&phrase, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(nil, &phrase, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(nil, &phrase, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE { + max := $6 + q := NewNumberRangeExpr(nil, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE { + min := $4 + q := NewNumberRangeExpr(&min, nil, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +}; + +posOrNegNumber: +tNUMBER { + $$ = $1 +} +| +tMINUS tNUMBER { + $$ = "-" + $2 +}; diff --git a/pkg/unify-query/internal/querystring/querystring.y.go b/pkg/unify-query/internal/querystring/querystring.y.go new file mode 100644 index 000000000..86e619b3c --- /dev/null +++ b/pkg/unify-query/internal/querystring/querystring.y.go @@ -0,0 +1,816 @@ +// Code generated by goyacc -o querystring.y.go querystring.y. DO NOT EDIT. + +//line querystring.y:2 +package querystring + +import __yyfmt__ "fmt" + +//line querystring.y:2 + +//line querystring.y:5 +type yySymType struct { + yys int + s string + n int + e Expr +} + +const tSTRING = 57346 +const tPHRASE = 57347 +const tNUMBER = 57348 +const tSLASH = 57349 +const tSTAR = 57350 +const tOR = 57351 +const tAND = 57352 +const tNOT = 57353 +const tTO = 57354 +const tPLUS = 57355 +const tMINUS = 57356 +const tCOLON = 57357 +const tLEFTBRACKET = 57358 +const tRIGHTBRACKET = 57359 +const tLEFTRANGE = 57360 +const tRIGHTRANGE = 57361 +const tLEFTBRACES = 57362 +const tRIGHTBRACES = 57363 +const tGREATER = 57364 +const tLESS = 57365 +const tEQUAL = 57366 + +var yyToknames = [...]string{ + "$end", + "error", + "$unk", + "tSTRING", + "tPHRASE", + "tNUMBER", + "tSLASH", + "tSTAR", + "tOR", + "tAND", + "tNOT", + "tTO", + "tPLUS", + "tMINUS", + "tCOLON", + "tLEFTBRACKET", + "tRIGHTBRACKET", + "tLEFTRANGE", + "tRIGHTRANGE", + "tLEFTBRACES", + "tRIGHTBRACES", + "tGREATER", + "tLESS", + "tEQUAL", +} + +var yyStatenames = [...]string{} + +const yyEofCode = 1 +const yyErrCode = 2 +const yyInitialStackSize = 16 + +//line yacctab:1 +var yyExca = [...]int8{ + -1, 1, + 1, -1, + -2, 0, +} + +const yyPrivate = 57344 + +const yyLast = 93 + +var yyAct = [...]int8{ + 28, 26, 29, 35, 30, 43, 35, 40, 35, 73, + 71, 36, 70, 27, 36, 33, 36, 34, 72, 31, + 32, 22, 67, 69, 42, 68, 39, 66, 50, 3, + 25, 59, 38, 41, 45, 47, 19, 35, 35, 61, + 51, 46, 35, 53, 44, 36, 36, 23, 24, 58, + 36, 12, 14, 13, 15, 57, 60, 62, 7, 64, + 10, 11, 56, 6, 54, 35, 52, 35, 48, 35, + 55, 17, 18, 36, 9, 36, 4, 36, 12, 14, + 13, 15, 49, 21, 20, 65, 63, 37, 2, 1, + 8, 5, 16, +} + +var yyPact = [...]int16{ + 47, -1000, -1000, 47, 62, -1000, 47, 47, 74, -1000, + -1000, -1000, 6, -1000, -1000, -1000, -1000, 47, 47, 13, + -1000, -1000, -3, -1000, -1000, -1000, -1000, 83, -1000, -1000, + -1000, 2, 0, 36, 63, -1000, 76, 11, -1000, 61, + -1000, -1000, 59, -1000, 58, 50, 43, 37, 19, -1000, + -1000, -1000, -1000, -1000, -1000, 32, 31, 81, 32, 80, + 8, 3, 4, -9, -1, -10, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, +} + +var yyPgo = [...]int8{ + 0, 0, 74, 88, 91, 29, 76, 90, 89, +} + +var yyR1 = [...]int8{ + 0, 8, 3, 3, 5, 5, 5, 6, 6, 6, + 4, 4, 7, 7, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, + 1, +} + +var yyR2 = [...]int8{ + 0, 1, 2, 1, 1, 3, 3, 1, 3, 2, + 2, 1, 1, 1, 1, 1, 1, 1, 3, 5, + 3, 3, 3, 4, 5, 4, 5, 4, 5, 4, + 5, 7, 7, 7, 7, 7, 7, 7, 7, 1, + 2, +} + +var yyChk = [...]int16{ + -1000, -8, -3, -5, -6, -4, 16, 11, -7, -2, + 13, 14, 4, 6, 5, 7, -3, 9, 10, -5, + -6, -2, 15, -5, -5, 17, 4, 16, -1, 5, + 7, 22, 23, 18, 20, 6, 14, 4, -1, 24, + 5, -1, 24, 5, 8, -1, 5, -1, 5, 6, + 17, -1, 5, -1, 5, 12, 12, 12, 12, 12, + -1, 8, -1, 5, -1, 5, 19, 19, 21, 19, + 21, 19, 19, 19, +} + +var yyDef = [...]int8{ + 0, -2, 1, 3, 4, 7, 0, 0, 0, 11, + 12, 13, 14, 15, 16, 17, 2, 0, 0, 0, + 9, 10, 0, 5, 6, 8, 18, 0, 20, 21, + 22, 0, 0, 0, 0, 39, 0, 0, 23, 0, + 27, 25, 0, 29, 0, 0, 0, 0, 0, 40, + 19, 24, 28, 26, 30, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 31, 32, 33, 37, + 34, 38, 35, 36, +} + +var yyTok1 = [...]int8{ + 1, +} + +var yyTok2 = [...]int8{ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, +} + +var yyTok3 = [...]int8{ + 0, +} + +var yyErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + yyDebug = 0 + yyErrorVerbose = false +) + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyParser interface { + Parse(yyLexer) int + Lookahead() int +} + +type yyParserImpl struct { + lval yySymType + stack [yyInitialStackSize]yySymType + char int +} + +func (p *yyParserImpl) Lookahead() int { + return p.char +} + +func yyNewParser() yyParser { + return &yyParserImpl{} +} + +const yyFlag = -1000 + +func yyTokname(c int) string { + if c >= 1 && c-1 < len(yyToknames) { + if yyToknames[c-1] != "" { + return yyToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func yyStatname(s int) string { + if s >= 0 && s < len(yyStatenames) { + if yyStatenames[s] != "" { + return yyStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func yyErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !yyErrorVerbose { + return "syntax error" + } + + for _, e := range yyErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + yyTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := int(yyPact[state]) + for tok := TOKSTART; tok-1 < len(yyToknames); tok++ { + if n := base + tok; n >= 0 && n < yyLast && int(yyChk[int(yyAct[n])]) == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if yyDef[state] == -2 { + i := 0 + for yyExca[i] != -1 || int(yyExca[i+1]) != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; yyExca[i] >= 0; i += 2 { + tok := int(yyExca[i]) + if tok < TOKSTART || yyExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if yyExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += yyTokname(tok) + } + return res +} + +func yylex1(lex yyLexer, lval *yySymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = int(yyTok1[0]) + goto out + } + if char < len(yyTok1) { + token = int(yyTok1[char]) + goto out + } + if char >= yyPrivate { + if char < yyPrivate+len(yyTok2) { + token = int(yyTok2[char-yyPrivate]) + goto out + } + } + for i := 0; i < len(yyTok3); i += 2 { + token = int(yyTok3[i+0]) + if token == char { + token = int(yyTok3[i+1]) + goto out + } + } + +out: + if token == 0 { + token = int(yyTok2[1]) /* unknown char */ + } + if yyDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char)) + } + return char, token +} + +func yyParse(yylex yyLexer) int { + return yyNewParser().Parse(yylex) +} + +func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int { + var yyn int + var yyVAL yySymType + var yyDollar []yySymType + _ = yyDollar // silence set and not used + yyS := yyrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yystate := 0 + yyrcvr.char = -1 + yytoken := -1 // yyrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + yystate = -1 + yyrcvr.char = -1 + yytoken = -1 + }() + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + if yyDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate)) + } + + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + yyn = int(yyPact[yystate]) + if yyn <= yyFlag { + goto yydefault /* simple state */ + } + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + yyn += yytoken + if yyn < 0 || yyn >= yyLast { + goto yydefault + } + yyn = int(yyAct[yyn]) + if int(yyChk[yyn]) == yytoken { /* valid shift */ + yyrcvr.char = -1 + yytoken = -1 + yyVAL = yyrcvr.lval + yystate = yyn + if Errflag > 0 { + Errflag-- + } + goto yystack + } + +yydefault: + /* default state action */ + yyn = int(yyDef[yystate]) + if yyn == -2 { + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if yyExca[xi+0] == -1 && int(yyExca[xi+1]) == yystate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + yyn = int(yyExca[xi+0]) + if yyn < 0 || yyn == yytoken { + break + } + } + yyn = int(yyExca[xi+1]) + if yyn < 0 { + goto ret0 + } + } + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + yylex.Error(yyErrorMessage(yystate, yytoken)) + Nerrs++ + if yyDebug >= 1 { + __yyfmt__.Printf("%s", yyStatname(yystate)) + __yyfmt__.Printf(" saw %s\n", yyTokname(yytoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + yyn = int(yyPact[yyS[yyp].yys]) + yyErrCode + if yyn >= 0 && yyn < yyLast { + yystate = int(yyAct[yyn]) /* simulate a shift of "error" */ + if int(yyChk[yystate]) == yyErrCode { + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken)) + } + if yytoken == yyEofCode { + goto ret1 + } + yyrcvr.char = -1 + yytoken = -1 + goto yynewstate /* try again in the same state */ + } + } + + /* reduction by production yyn */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + } + + yynt := yyn + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= int(yyR2[yyn]) + // yyp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + yyn = int(yyR1[yyn]) + yyg := int(yyPgo[yyn]) + yyj := yyg + yyS[yyp].yys + 1 + + if yyj >= yyLast { + yystate = int(yyAct[yyg]) + } else { + yystate = int(yyAct[yyj]) + if int(yyChk[yystate]) != -yyn { + yystate = int(yyAct[yyg]) + } + } + // dummy call; replaced with literal code + switch yynt { + + case 1: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:32 + { + yylex.(*lexerWrapper).expr = yyDollar[1].e + } + case 2: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:37 + { + yyVAL.e = NewOrExpr(yyDollar[1].e, yyDollar[2].e) + } + case 3: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:41 + { + yyVAL.e = yyDollar[1].e + } + case 4: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:46 + { + yyVAL.e = yyDollar[1].e + } + case 5: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:50 + { + yyVAL.e = NewOrExpr(yyDollar[1].e, yyDollar[3].e) + } + case 6: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:54 + { + yyVAL.e = NewAndExpr(yyDollar[1].e, yyDollar[3].e) + } + case 7: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:59 + { + yyVAL.e = yyDollar[1].e + } + case 8: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:63 + { + yyVAL.e = yyDollar[2].e + } + case 9: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:67 + { + yyVAL.e = NewNotExpr(yyDollar[2].e) + } + case 10: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:72 + { + switch yyDollar[1].n { + case queryMustNot: + yyVAL.e = NewNotExpr(yyDollar[2].e) + default: + yyVAL.e = yyDollar[2].e + } + } + case 11: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:81 + { + yyVAL.e = yyDollar[1].e + } + case 12: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:86 + { + yyVAL.n = queryMust + } + case 13: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:90 + { + yyVAL.n = queryMustNot + } + case 14: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:95 + { + yyVAL.e = newStringExpr(yyDollar[1].s) + } + case 15: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:99 + { + yyVAL.e = NewMatchExpr(yyDollar[1].s) + } + case 16: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:103 + { + phrase := yyDollar[1].s + q := NewMatchExpr(phrase) + yyVAL.e = q + } + case 17: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:109 + { + phrase := yyDollar[1].s + q := NewRegexpExpr(phrase) + yyVAL.e = q + } + case 18: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:115 + { + q := newStringExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 19: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:121 + { + q := newStringExpr(yyDollar[4].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 20: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:127 + { + q := NewMatchExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 21: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:133 + { + q := NewMatchExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 22: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:139 + { + q := NewRegexpExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 23: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:145 + { + val := yyDollar[4].s + q := NewNumberRangeExpr(&val, nil, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 24: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:152 + { + val := yyDollar[5].s + q := NewNumberRangeExpr(&val, nil, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 25: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:159 + { + val := yyDollar[4].s + q := NewNumberRangeExpr(nil, &val, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 26: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:166 + { + val := yyDollar[5].s + q := NewNumberRangeExpr(nil, &val, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 27: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:173 + { + phrase := yyDollar[4].s + q := NewTimeRangeExpr(&phrase, nil, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 28: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:180 + { + phrase := yyDollar[5].s + q := NewTimeRangeExpr(&phrase, nil, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 29: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:187 + { + phrase := yyDollar[4].s + q := NewTimeRangeExpr(nil, &phrase, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 30: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:194 + { + phrase := yyDollar[5].s + q := NewTimeRangeExpr(nil, &phrase, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 31: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:201 + { + max := yyDollar[6].s + q := NewNumberRangeExpr(nil, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 32: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:208 + { + min := yyDollar[4].s + q := NewNumberRangeExpr(&min, nil, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 33: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:215 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 34: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:223 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 35: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:231 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 36: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:239 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 37: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:247 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 38: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:255 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 39: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:264 + { + yyVAL.s = yyDollar[1].s + } + case 40: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:268 + { + yyVAL.s = "-" + yyDollar[2].s + } + } + goto yystack /* stack new state and value */ +} diff --git a/pkg/unify-query/internal/querystring/y.output b/pkg/unify-query/internal/querystring/y.output new file mode 100644 index 000000000..7c82715fc --- /dev/null +++ b/pkg/unify-query/internal/querystring/y.output @@ -0,0 +1,679 @@ + +state 0 + $accept: .input $end + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchLogicParts goto 2 + searchPart goto 5 + searchLogicPart goto 3 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + input goto 1 + +state 1 + $accept: input.$end + + $end accept + . error + + +state 2 + input: searchLogicParts. (1) + + . reduce 1 (src line 31) + + +state 3 + searchLogicParts: searchLogicPart.searchLogicParts + searchLogicParts: searchLogicPart. (3) + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . reduce 3 (src line 40) + + searchBase goto 9 + searchLogicParts goto 16 + searchPart goto 5 + searchLogicPart goto 3 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 4 + searchLogicPart: searchLogicSimplePart. (4) + searchLogicPart: searchLogicSimplePart.tOR searchLogicPart + searchLogicPart: searchLogicSimplePart.tAND searchLogicPart + + tOR shift 17 + tAND shift 18 + . reduce 4 (src line 45) + + +state 5 + searchLogicSimplePart: searchPart. (7) + + . reduce 7 (src line 58) + + +state 6 + searchLogicSimplePart: tLEFTBRACKET.searchLogicPart tRIGHTBRACKET + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 19 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 7 + searchLogicSimplePart: tNOT.searchLogicSimplePart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicSimplePart goto 20 + searchPrefix goto 8 + +state 8 + searchPart: searchPrefix.searchBase + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + . error + + searchBase goto 21 + +state 9 + searchPart: searchBase. (11) + + . reduce 11 (src line 80) + + +state 10 + searchPrefix: tPLUS. (12) + + . reduce 12 (src line 85) + + +state 11 + searchPrefix: tMINUS. (13) + + . reduce 13 (src line 89) + + +state 12 + searchBase: tSTRING. (14) + searchBase: tSTRING.tCOLON tSTRING + searchBase: tSTRING.tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET + searchBase: tSTRING.tCOLON posOrNegNumber + searchBase: tSTRING.tCOLON tPHRASE + searchBase: tSTRING.tCOLON tSLASH + searchBase: tSTRING.tCOLON tGREATER posOrNegNumber + searchBase: tSTRING.tCOLON tGREATER tEQUAL posOrNegNumber + searchBase: tSTRING.tCOLON tLESS posOrNegNumber + searchBase: tSTRING.tCOLON tLESS tEQUAL posOrNegNumber + searchBase: tSTRING.tCOLON tGREATER tPHRASE + searchBase: tSTRING.tCOLON tGREATER tEQUAL tPHRASE + searchBase: tSTRING.tCOLON tLESS tPHRASE + searchBase: tSTRING.tCOLON tLESS tEQUAL tPHRASE + searchBase: tSTRING.tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING.tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING.tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE + + tCOLON shift 22 + . reduce 14 (src line 94) + + +state 13 + searchBase: tNUMBER. (15) + + . reduce 15 (src line 98) + + +state 14 + searchBase: tPHRASE. (16) + + . reduce 16 (src line 102) + + +state 15 + searchBase: tSLASH. (17) + + . reduce 17 (src line 108) + + +state 16 + searchLogicParts: searchLogicPart searchLogicParts. (2) + + . reduce 2 (src line 36) + + +state 17 + searchLogicPart: searchLogicSimplePart tOR.searchLogicPart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 23 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 18 + searchLogicPart: searchLogicSimplePart tAND.searchLogicPart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 24 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 19 + searchLogicSimplePart: tLEFTBRACKET searchLogicPart.tRIGHTBRACKET + + tRIGHTBRACKET shift 25 + . error + + +state 20 + searchLogicSimplePart: tNOT searchLogicSimplePart. (9) + + . reduce 9 (src line 66) + + +state 21 + searchPart: searchPrefix searchBase. (10) + + . reduce 10 (src line 71) + + +state 22 + searchBase: tSTRING tCOLON.tSTRING + searchBase: tSTRING tCOLON.tLEFTBRACKET tSTRING tRIGHTBRACKET + searchBase: tSTRING tCOLON.posOrNegNumber + searchBase: tSTRING tCOLON.tPHRASE + searchBase: tSTRING tCOLON.tSLASH + searchBase: tSTRING tCOLON.tGREATER posOrNegNumber + searchBase: tSTRING tCOLON.tGREATER tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON.tLESS posOrNegNumber + searchBase: tSTRING tCOLON.tLESS tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON.tGREATER tPHRASE + searchBase: tSTRING tCOLON.tGREATER tEQUAL tPHRASE + searchBase: tSTRING tCOLON.tLESS tPHRASE + searchBase: tSTRING tCOLON.tLESS tEQUAL tPHRASE + searchBase: tSTRING tCOLON.tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON.tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON.tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE + + tSTRING shift 26 + tPHRASE shift 29 + tNUMBER shift 35 + tSLASH shift 30 + tMINUS shift 36 + tLEFTBRACKET shift 27 + tLEFTRANGE shift 33 + tLEFTBRACES shift 34 + tGREATER shift 31 + tLESS shift 32 + . error + + posOrNegNumber goto 28 + +state 23 + searchLogicPart: searchLogicSimplePart tOR searchLogicPart. (5) + + . reduce 5 (src line 49) + + +state 24 + searchLogicPart: searchLogicSimplePart tAND searchLogicPart. (6) + + . reduce 6 (src line 53) + + +state 25 + searchLogicSimplePart: tLEFTBRACKET searchLogicPart tRIGHTBRACKET. (8) + + . reduce 8 (src line 62) + + +state 26 + searchBase: tSTRING tCOLON tSTRING. (18) + + . reduce 18 (src line 114) + + +state 27 + searchBase: tSTRING tCOLON tLEFTBRACKET.tSTRING tRIGHTBRACKET + + tSTRING shift 37 + . error + + +state 28 + searchBase: tSTRING tCOLON posOrNegNumber. (20) + + . reduce 20 (src line 126) + + +state 29 + searchBase: tSTRING tCOLON tPHRASE. (21) + + . reduce 21 (src line 132) + + +state 30 + searchBase: tSTRING tCOLON tSLASH. (22) + + . reduce 22 (src line 138) + + +state 31 + searchBase: tSTRING tCOLON tGREATER.posOrNegNumber + searchBase: tSTRING tCOLON tGREATER.tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON tGREATER.tPHRASE + searchBase: tSTRING tCOLON tGREATER.tEQUAL tPHRASE + + tPHRASE shift 40 + tNUMBER shift 35 + tMINUS shift 36 + tEQUAL shift 39 + . error + + posOrNegNumber goto 38 + +state 32 + searchBase: tSTRING tCOLON tLESS.posOrNegNumber + searchBase: tSTRING tCOLON tLESS.tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON tLESS.tPHRASE + searchBase: tSTRING tCOLON tLESS.tEQUAL tPHRASE + + tPHRASE shift 43 + tNUMBER shift 35 + tMINUS shift 36 + tEQUAL shift 42 + . error + + posOrNegNumber goto 41 + +state 33 + searchBase: tSTRING tCOLON tLEFTRANGE.tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE.tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.tPHRASE tTO tPHRASE tRIGHTRANGE + + tPHRASE shift 46 + tNUMBER shift 35 + tSTAR shift 44 + tMINUS shift 36 + . error + + posOrNegNumber goto 45 + +state 34 + searchBase: tSTRING tCOLON tLEFTBRACES.posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTBRACES.tPHRASE tTO tPHRASE tRIGHTRANGE + + tPHRASE shift 48 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 47 + +state 35 + posOrNegNumber: tNUMBER. (39) + + . reduce 39 (src line 263) + + +state 36 + posOrNegNumber: tMINUS.tNUMBER + + tNUMBER shift 49 + . error + + +state 37 + searchBase: tSTRING tCOLON tLEFTBRACKET tSTRING.tRIGHTBRACKET + + tRIGHTBRACKET shift 50 + . error + + +state 38 + searchBase: tSTRING tCOLON tGREATER posOrNegNumber. (23) + + . reduce 23 (src line 144) + + +state 39 + searchBase: tSTRING tCOLON tGREATER tEQUAL.posOrNegNumber + searchBase: tSTRING tCOLON tGREATER tEQUAL.tPHRASE + + tPHRASE shift 52 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 51 + +state 40 + searchBase: tSTRING tCOLON tGREATER tPHRASE. (27) + + . reduce 27 (src line 172) + + +state 41 + searchBase: tSTRING tCOLON tLESS posOrNegNumber. (25) + + . reduce 25 (src line 158) + + +state 42 + searchBase: tSTRING tCOLON tLESS tEQUAL.posOrNegNumber + searchBase: tSTRING tCOLON tLESS tEQUAL.tPHRASE + + tPHRASE shift 54 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 53 + +state 43 + searchBase: tSTRING tCOLON tLESS tPHRASE. (29) + + . reduce 29 (src line 186) + + +state 44 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 55 + . error + + +state 45 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 56 + . error + + +state 46 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE.tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE.tTO tPHRASE tRIGHTRANGE + + tTO shift 57 + . error + + +state 47 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 58 + . error + + +state 48 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE.tTO tPHRASE tRIGHTRANGE + + tTO shift 59 + . error + + +state 49 + posOrNegNumber: tMINUS tNUMBER. (40) + + . reduce 40 (src line 267) + + +state 50 + searchBase: tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET. (19) + + . reduce 19 (src line 120) + + +state 51 + searchBase: tSTRING tCOLON tGREATER tEQUAL posOrNegNumber. (24) + + . reduce 24 (src line 151) + + +state 52 + searchBase: tSTRING tCOLON tGREATER tEQUAL tPHRASE. (28) + + . reduce 28 (src line 179) + + +state 53 + searchBase: tSTRING tCOLON tLESS tEQUAL posOrNegNumber. (26) + + . reduce 26 (src line 165) + + +state 54 + searchBase: tSTRING tCOLON tLESS tEQUAL tPHRASE. (30) + + . reduce 30 (src line 193) + + +state 55 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 60 + +state 56 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tSTAR shift 61 + tMINUS shift 36 + . error + + posOrNegNumber goto 62 + +state 57 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO.tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO.tPHRASE tRIGHTRANGE + + tPHRASE shift 63 + . error + + +state 58 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 64 + +state 59 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO.tPHRASE tRIGHTRANGE + + tPHRASE shift 65 + . error + + +state 60 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 66 + . error + + +state 61 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR.tRIGHTRANGE + + tRIGHTRANGE shift 67 + . error + + +state 62 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber.tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 69 + tRIGHTBRACES shift 68 + . error + + +state 63 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE.tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE.tRIGHTRANGE + + tRIGHTRANGE shift 71 + tRIGHTBRACES shift 70 + . error + + +state 64 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 72 + . error + + +state 65 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE.tRIGHTRANGE + + tRIGHTRANGE shift 73 + . error + + +state 66 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE. (31) + + . reduce 31 (src line 200) + + +state 67 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE. (32) + + . reduce 32 (src line 207) + + +state 68 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES. (33) + + . reduce 33 (src line 214) + + +state 69 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE. (37) + + . reduce 37 (src line 246) + + +state 70 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES. (34) + + . reduce 34 (src line 222) + + +state 71 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE. (38) + + . reduce 38 (src line 254) + + +state 72 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE. (35) + + . reduce 35 (src line 230) + + +state 73 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE. (36) + + . reduce 36 (src line 238) + + +24 terminals, 9 nonterminals +41 grammar rules, 74/16000 states +0 shift/reduce, 0 reduce/reduce conflicts reported +58 working sets used +memory: parser 42/240000 +12 extra closures +112 shift entries, 1 exceptions +23 goto entries +20 entries saved by goto default +Optimizer space used: output 93/240000 +93 table entries, 0 zero +maximum spread: 24, maximum offset: 58 diff --git a/pkg/unify-query/kvstore/bbolt/bbolt.go b/pkg/unify-query/kvstore/bbolt/bbolt.go index a1029e6d4..c41ce39b8 100644 --- a/pkg/unify-query/kvstore/bbolt/bbolt.go +++ b/pkg/unify-query/kvstore/bbolt/bbolt.go @@ -101,6 +101,10 @@ func (c *Client) Open() error { return fmt.Errorf("unable to open boltdb: %w", err) } + if db == nil { + return fmt.Errorf("open boltdb is empty") + } + // New Bucket err = db.Update(func(tx *bolt.Tx) error { _, bucketErr := tx.CreateBucketIfNotExists(c.BucketName) diff --git a/pkg/unify-query/log/hook.go b/pkg/unify-query/log/hook.go index c89a2ae9b..cbcc05f46 100644 --- a/pkg/unify-query/log/hook.go +++ b/pkg/unify-query/log/hook.go @@ -113,6 +113,6 @@ func init() { // InitTestLogger 加载单元测试日志配置 func InitTestLogger() { // 加载配置 - viper.Set(LevelConfigPath, "warn") + viper.Set(LevelConfigPath, "debug") initLogConfig() } diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index ce9aa44fc..49ceb2460 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -29,24 +29,44 @@ const ( UUID = "query_uuid" ) +const ( + ConditionEqual = "eq" + ConditionNotEqual = "ne" + ConditionRegEqual = "req" + ConditionNotRegEqual = "nreq" + ConditionContains = "contains" + ConditionNotContains = "ncontains" + + ConditionExisted = "existed" + ConditionNotExisted = "nexisted" + + ConditionExact = "exact" + ConditionGt = "gt" + ConditionGte = "gte" + ConditionLt = "lt" + ConditionLte = "lte" +) + type VmCondition string type TimeField struct { - Name string - Type string - Unit string + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Unit string `json:"unit,omitempty"` } // Aggregate 聚合方法 type Aggregate struct { - Name string - Dimensions []string - Without bool + Name string `json:"name,omitempty"` + Field string `json:"field,omitempty"` + + Dimensions []string `json:"dimensions,omitempty"` + Without bool `json:"without,omitempty"` - Window time.Duration - TimeZone string + Window time.Duration `json:"window,omitempty"` + TimeZone string `json:"time_zone,omitempty"` - Args []interface{} + Args []interface{} `json:"args,omitempty"` } // OffSetInfo Offset的信息存储，供promql查询转换为influxdb查询语句时使用 @@ -61,73 +81,73 @@ type Aggregates []Aggregate // Query 查询扩展信息，为后面查询提供定位 type Query struct { - SourceType string - Password string // 查询鉴权 + SourceType string `json:"source_type,omitempty"` + Password string `json:"password,omitempty"` // 查询鉴权 - ClusterID string // 存储 ID + ClusterID string `json:"cluster_id,omitempty"` // 存储 ID - StorageType string // 存储类型 + StorageType string `json:"storage_type,omitempty"` // 存储类型 - StorageIDs []string - StorageID string - StorageName string + StorageIDs []string `json:"storage_ids,omitempty"` + StorageID string `json:"storage_id,omitempty"` + StorageName string `json:"storage_name,omitempty"` - ClusterName string - TagsKey []string + ClusterName string `json:"cluster_name,omitempty"` + TagsKey []string `json:"tags_key,omitempty"` - DataSource string - DataLabel string - TableID string - MetricName string + DataSource string `json:"data_source,omitempty"` + DataLabel string `json:"data_label,omitempty"` + TableID string `json:"table_id,omitempty"` + MetricName string `json:"metric_name,omitempty"` // vm 的 rt - VmRt string + VmRt string `json:"vm_rt,omitempty"` // 兼容 InfluxDB 结构体 - RetentionPolicy string // 存储 RP - DB string // 存储 DB - Measurement string // 存储 Measurement - Field string // 存储 Field - TimeField TimeField // 时间字段 - Timezone string // 存储 Timezone - Fields []string // 存储命中的 Field 列表，一般情况下为一个，当 Field 为模糊匹配时，解析为多个 - Measurements []string // 存储命中的 Measurement 列表，一般情况下为一个，当 Measurement 为模糊匹配时，解析为多个 + RetentionPolicy string `json:"retention_policy,omitempty"` // 存储 RP + DB string `json:"db,omitempty"` // 存储 DB + Measurement string `json:"measurement,omitempty"` // 存储 Measurement + Field string `json:"field,omitempty"` // 存储 Field + TimeField TimeField `json:"time_field,omitempty"` // 时间字段 + Timezone string `json:"timezone,omitempty"` // 存储 Timezone + Fields []string `json:"fields,omitempty"` // 存储命中的 Field 列表，一般情况下为一个，当 Field 为模糊匹配时，解析为多个 + Measurements []string `json:"measurements,omitempty"` // 存储命中的 Measurement 列表，一般情况下为一个，当 Measurement 为模糊匹配时，解析为多个 // 用于 promql 查询 - IsHasOr bool // 标记是否有 or 条件 + IsHasOr bool `json:"is_has_or,omitempty"` // 标记是否有 or 条件 - Aggregates Aggregates // 聚合方法列表，从内到外排序 + Aggregates Aggregates `json:"aggregates,omitempty"` // 聚合方法列表，从内到外排序 - Condition string // 过滤条件 + Condition string `json:"condition,omitempty"` // 过滤条件 // BkSql 过滤条件 - BkSqlCondition string + BkSqlCondition string `json:"bk_sql_condition,omitempty"` // Vm 过滤条件 - VmCondition VmCondition - VmConditionNum int + VmCondition VmCondition `json:"vm_condition,omitempty"` + VmConditionNum int `json:"vm_condition_num,omitempty"` - Filters []map[string]string // 查询中自带查询条件，用于拼接 + Filters []map[string]string `json:"filters,omitempty"` // 查询中自带查询条件，用于拼接 - OffsetInfo OffSetInfo // limit等偏移量配置 + OffsetInfo OffSetInfo `json:"offset_info,omitempty"` // limit等偏移量配置 - SegmentedEnable bool // 是否开启分段查询 + SegmentedEnable bool `json:"segmented_enable,omitempty"` // 是否开启分段查询 // Es 查询扩展 - QueryString string - AllConditions AllConditions + QueryString string `json:"query_string,omitempty"` + AllConditions AllConditions `json:"all_conditions,omitempty"` - HighLight *HighLight + HighLight *HighLight `json:"high_light,omitempty"` - Source []string - From int - Size int + Source []string `json:"source,omitempty"` + From int `json:"from,omitempty"` + Size int `json:"size,omitempty"` - Scroll string - ResultTableOptions ResultTableOptions + Scroll string `json:"scroll,omitempty"` + ResultTableOptions ResultTableOptions `json:"result_table_options,omitempty"` - Orders Orders - NeedAddTime bool + Orders Orders `json:"orders,omitempty"` + NeedAddTime bool `json:"need_add_time,omitempty"` } type HighLight struct { @@ -255,10 +275,13 @@ func ReplaceVmCondition(condition VmCondition, replaceLabels ReplaceLabels) VmCo } // ToJson 通过 tableID 排序，并且返回 json 序列化 -func (qMetric QueryMetric) ToJson(isSort bool) string { +func (qMetric *QueryMetric) ToJson(isSort bool) string { if isSort { - sort.SliceIsSorted(qMetric.QueryList, func(i, j int) bool { - return qMetric.QueryList[i].TableID < qMetric.QueryList[j].TableID + sort.SliceStable(qMetric.QueryList, func(i, j int) bool { + a := qMetric.QueryList[i].TableID + b := qMetric.QueryList[j].TableID + c := a < b + return c }) } diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index 6d9b51c07..417cfa9d8 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -158,16 +158,23 @@ type elasticSearchResultData struct { func mockHandler(ctx context.Context) { httpmock.Activate() - mockVmHandler(ctx) + log.Infof(context.Background(), "mock handler start") + + mockBKBaseHandler(ctx) mockInfluxDBHandler(ctx) - mockBkSQLHandler(ctx) mockElasticSearchHandler(ctx) + + log.Infof(context.Background(), "mock handler end") } const ( - EsUrl = "http://127.0.0.1:93002" - BkSQLUrl = "http://127.0.0.1:92001" - VmUrl = "http://127.0.0.1:12001/bk_data/query_sync" + EsUrlDomain = "http://127.0.0.1:93002" + BkBaseUrlDomain = "http://127.0.0.1:12001" +) + +const ( + EsUrl = EsUrlDomain + BkBaseUrl = BkBaseUrlDomain + "/bk_data/query_sync" ) type BkSQLRequest struct { @@ -179,7 +186,7 @@ type BkSQLRequest struct { } func mockElasticSearchHandler(ctx context.Context) { - bkBaseUrl := "http://127.0.0.1:12001/bk_data/query_sync/es" + bkBaseEsUrl := BkBaseUrl + "/es" searchHandler := func(r *http.Request) (w *http.Response, err error) { body, _ := io.ReadAll(r.Body) @@ -196,45 +203,20 @@ func mockElasticSearchHandler(ctx context.Context) { mappings := `{"es_index":{"mappings":{"properties":{"a":{"type":"keyword"},"b":{"type":"keyword"},"group":{"type":"keyword"},"kibana_stats":{"properties":{"kibana":{"properties":{"name":{"type":"keyword"}}}}},"timestamp":{"type":"log"},"type":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` mappingResp := httpmock.NewStringResponder(http.StatusOK, mappings) - httpmock.RegisterResponder(http.MethodGet, bkBaseUrl+"/es_index/_mapping/", mappingResp) + httpmock.RegisterResponder(http.MethodGet, bkBaseEsUrl+"/es_index/_mapping/", mappingResp) httpmock.RegisterResponder(http.MethodGet, EsUrl+"/es_index/_mapping/", mappingResp) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/es_index/_search", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search", searchHandler) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search?scroll=5m", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/es_index/_search?scroll=5m", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search?scroll=5m", searchHandler) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/_search/scroll", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/_search/scroll", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/_search/scroll", searchHandler) - httpmock.RegisterResponder(http.MethodHead, EsUrl, searchHandler) -} - -func mockBkSQLHandler(ctx context.Context) { - httpmock.RegisterResponder(http.MethodPost, BkSQLUrl, func(r *http.Request) (w *http.Response, err error) { - var ( - request BkSQLRequest - ) - err = json.NewDecoder(r.Body).Decode(&request) - if err != nil { - return - } - - d, ok := BkSQL.Get(request.Sql) - if !ok { - err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) - log.Errorf(ctx, err.Error()) - return - } - - switch t := d.(type) { - case string: - w = httpmock.NewStringResponse(http.StatusOK, t) - default: - w, err = httpmock.NewJsonResponse(http.StatusOK, d) - } - return + httpmock.RegisterResponder(http.MethodHead, EsUrl, func(request *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(http.StatusOK, ""), nil }) } @@ -268,8 +250,8 @@ func mockInfluxDBHandler(ctx context.Context) { }) } -func mockVmHandler(ctx context.Context) { - httpmock.RegisterResponder(http.MethodPost, VmUrl, func(r *http.Request) (w *http.Response, err error) { +func mockBKBaseHandler(ctx context.Context) { + httpmock.RegisterResponder(http.MethodPost, BkBaseUrl, func(r *http.Request) (w *http.Response, err error) { var ( request VmRequest params VmParams @@ -279,6 +261,23 @@ func mockVmHandler(ctx context.Context) { return } + if request.PreferStorage != "vm" { + d, ok := BkSQL.Get(request.Sql) + if !ok { + err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) + log.Errorf(ctx, err.Error()) + return + } + switch t := d.(type) { + case string: + w = httpmock.NewStringResponse(http.StatusOK, t) + default: + w, err = httpmock.NewJsonResponse(http.StatusOK, d) + } + + return + } + err = json.Unmarshal([]byte(request.Sql), ¶ms) if err != nil { return @@ -306,7 +305,6 @@ func mockVmHandler(ctx context.Context) { d, ok := Vm.Get(key) if !ok { err = fmt.Errorf(`vm mock data is empty in "%s"`, key) - log.Errorf(ctx, err.Error()) return } diff --git a/pkg/unify-query/query/structured/condition.go b/pkg/unify-query/query/structured/condition.go index e06452aba..60b66e214 100644 --- a/pkg/unify-query/query/structured/condition.go +++ b/pkg/unify-query/query/structured/condition.go @@ -198,6 +198,28 @@ func MergeConditionField(source, target AllConditions) AllConditions { return all } +func (c AllConditions) MetaDataAllConditions() metadata.AllConditions { + var allConditions metadata.AllConditions + if len(c) > 0 { + allConditions = make(metadata.AllConditions, 0, len(c)) + for _, conditions := range c { + conds := make([]metadata.ConditionField, 0, len(conditions)) + for _, cond := range conditions { + conds = append(conds, metadata.ConditionField{ + DimensionName: cond.DimensionName, + Value: cond.Value, + Operator: cond.Operator, + IsWildcard: cond.IsWildcard, + IsPrefix: cond.IsPrefix, + IsSuffix: cond.IsSuffix, + }) + } + allConditions = append(allConditions, conds) + } + } + return allConditions +} + func (c AllConditions) BkSql() string { var conditionsString []string for _, cond := range c { diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index b13d911fb..9b4510e77 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -48,10 +48,9 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) for _, aggr := range a { agg := metadata.Aggregate{ Name: aggr.Method, - Dimensions: aggr.Dimensions, + Dimensions: append([]string{}, aggr.Dimensions...), Without: aggr.Without, Args: aggr.VArgsList, - TimeZone: timezone, } if aggr.Window != "" { @@ -61,6 +60,7 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) } agg.Window = time.Duration(window) + agg.TimeZone = timezone } aggs = append(aggs, agg) } @@ -71,6 +71,8 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) type AggregateMethod struct { // Method 聚合方法 Method string `json:"method,omitempty" example:"mean"` + // Field 聚合字段，默认为指标字段，指定则会进行覆盖 + Field string `json:"field,omitempty" example:"field"` // Without Without bool `json:"without" example:false` // Dimensions 聚合维度 diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index a1cb63a90..829c2826d 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -150,10 +150,8 @@ func (q *QueryTs) ToQueryReference(ctx context.Context) (metadata.QueryReference } // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if query.HighLight != nil { - if !query.HighLight.Enable && q.HighLight.Enable { - query.HighLight = q.HighLight - } + if query.HighLight == nil && q.HighLight != nil { + query.HighLight = q.HighLight } // 复用字段配置，没有特殊配置的情况下使用公共配置 @@ -187,26 +185,12 @@ func (q *QueryTs) ToQueryClusterMetric(ctx context.Context) (*metadata.QueryClus return nil, errors.Errorf("Only one query supported, now %d ", len(q.QueryList)) } - for _, qry = range q.QueryList { - } + qry = q.QueryList[0] // 结构定义转换 allConditions, err := qry.Conditions.AnalysisConditions() - queryConditions := make([][]metadata.ConditionField, 0, len(allConditions)) - for _, conds := range allConditions { - queryConds := make([]metadata.ConditionField, 0, len(conds)) - for _, cond := range conds { - queryConds = append(queryConds, metadata.ConditionField{ - DimensionName: cond.DimensionName, - Value: cond.Value, - Operator: cond.Operator, - IsWildcard: cond.IsWildcard, - IsPrefix: cond.IsPrefix, - IsSuffix: cond.IsSuffix, - }) - } - queryConditions = append(queryConditions, queryConds) - } + queryConditions := allConditions.MetaDataAllConditions() + if err != nil { return nil, err } @@ -466,7 +450,8 @@ func (q *Query) Aggregates() (aggs metadata.Aggregates, err error) { if name, ok := domSampledFunc[am.Method+q.TimeAggregation.Function]; ok { agg := metadata.Aggregate{ Name: name, - Dimensions: am.Dimensions, + Field: am.Field, + Dimensions: append([]string{}, am.Dimensions...), Without: am.Without, Window: time.Duration(window), TimeZone: q.Timezone, @@ -504,6 +489,11 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q return nil, err } + allConditions, err := q.Conditions.AnalysisConditions() + if err != nil { + return nil, err + } + // 如果 DataSource 为空，则自动补充 if q.DataSource == "" { q.DataSource = BkMonitor @@ -511,12 +501,6 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q // 如果是 BkSql 查询无需获取 tsdb 路由关系 if q.DataSource == BkData { - allConditions, bkDataErr := q.Conditions.AnalysisConditions() - if bkDataErr != nil { - err = bkDataErr - return nil, bkDataErr - } - // 判断空间跟业务是否匹配 isMatchBizID := func() bool { space := strings.Split(spaceUid, "__") @@ -552,15 +536,39 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q } qry := &metadata.Query{ - StorageType: consul.BkSqlStorageType, - TableID: string(tableID), - DataSource: q.DataSource, - DB: route.DB(), - Measurement: route.Measurement(), - Field: q.FieldName, - MetricName: metricName, - Aggregates: aggregates, - BkSqlCondition: allConditions.BkSql(), + StorageType: consul.BkSqlStorageType, + TableID: string(tableID), + DataSource: q.DataSource, + DB: route.DB(), + Measurement: route.Measurement(), + Field: q.FieldName, + MetricName: metricName, + Aggregates: aggregates, + AllConditions: allConditions.MetaDataAllConditions(), + Size: q.Limit, + From: q.From, + } + + if len(q.OrderBy) > 0 { + qry.Orders = make(metadata.Orders, 0, len(q.OrderBy)) + for _, o := range q.OrderBy { + if len(o) == 0 { + continue + } + + asc := true + name := o + + if strings.HasPrefix(o, "-") { + asc = false + name = name[1:] + } + + qry.Orders = append(qry.Orders, metadata.Order{ + Name: name, + Ast: asc, + }) + } } metadata.GetQueryParams(ctx).SetStorageType(qry.StorageType) @@ -580,11 +588,6 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q isSkipField = true } - allConditions, err := q.Conditions.AnalysisConditions() - if err != nil { - return nil, err - } - tsDBs, err := GetTsDBList(ctx, &TsDBOption{ SpaceUid: spaceUid, TableID: tableID, @@ -608,6 +611,7 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q span.Set("query-metric", metricName) span.Set("query-is-regexp", q.IsRegexp) span.Set("tsdb-num", len(tsDBs)) + span.Set("query-aggregate", aggregates) for _, tsDB := range tsDBs { query, buildErr := q.BuildMetadataQuery(ctx, tsDB, allConditions, queryLabelsMatcher) @@ -875,21 +879,19 @@ func (q *Query) BuildMetadataQuery( return nil, err } - // 在 metadata 还没有补充 storageType 字段之前 - // 使用 sourceType 来判断是否是 es 查询 - // 等后面支持了之后可以删除该段逻辑 - if tsDB.SourceType == BkData { - query.StorageType = consul.ElasticsearchStorageType - } - // 兼容原逻辑，storageType 通过 storageMap 获取 if query.StorageType == "" { + log.Warnf(ctx, "storageType is empty with %s", tsDB.TableID) stg, _ := tsdb.GetStorage(query.StorageID) if stg != nil { query.StorageType = stg.Type } } + if query.StorageType == "" { + return nil, fmt.Errorf("storageType is empty with %v", query.StorageID) + } + query.Measurement = measurement query.Measurements = measurements query.Field = field @@ -909,12 +911,14 @@ func (q *Query) BuildMetadataQuery( query.NeedAddTime = tsDB.NeedAddTime query.SourceType = tsDB.SourceType + query.AllConditions = allCondition.MetaDataAllConditions() query.Condition = whereList.String() query.VmCondition, query.VmConditionNum = allCondition.VMString(query.VmRt, vmMetric, q.IsRegexp) // 写入 ES 所需内容 query.QueryString = q.QueryString query.Source = q.KeepColumns + query.HighLight = q.HighLight query.Scroll = q.Scroll @@ -923,24 +927,6 @@ func (q *Query) BuildMetadataQuery( query.Size = q.Limit query.From = q.From - if len(allCondition) > 0 { - query.AllConditions = make(metadata.AllConditions, len(allCondition)) - for i, conditions := range allCondition { - conds := make([]metadata.ConditionField, len(conditions)) - for j, c := range conditions { - conds[j] = metadata.ConditionField{ - DimensionName: c.DimensionName, - Value: c.Value, - Operator: c.Operator, - IsWildcard: c.IsWildcard, - IsPrefix: c.IsPrefix, - IsSuffix: c.IsSuffix, - } - } - query.AllConditions[i] = conds - } - } - if len(q.OrderBy) > 0 { query.Orders = q.OrderBy.Orders() } diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index e8bca55e8..54ee67f66 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -11,7 +11,6 @@ package structured import ( "context" - "encoding/json" "testing" "time" @@ -272,10 +271,7 @@ func TestQueryToMetric(t *testing.T) { metric, err := c.query.ToQueryMetric(ctx, spaceUID) assert.Nil(t, err) - a, _ := json.Marshal(c.metric) - b, _ := json.Marshal(metric) - - assert.JSONEq(t, string(a), string(b)) + assert.Equal(t, c.metric.ToJson(true), metric.ToJson(true)) }) } } @@ -1141,7 +1137,6 @@ func TestQueryTs_ToQueryReference(t *testing.T) { { DataSource: BkLog, Timezone: "UTC", - SourceType: "bkdata", TableID: "result_table.es", DataLabel: "es", DB: "es_index", @@ -1169,6 +1164,81 @@ func TestQueryTs_ToQueryReference(t *testing.T) { }, }, }, + "es 高亮查询": { + ts: &QueryTs{ + QueryList: []*Query{ + { + DataSource: BkLog, + TableID: "result_table.es", + FieldName: "usage", + ReferenceName: "a", + TimeAggregation: TimeAggregation{ + Function: "sum_over_time", + Window: "1m", + }, + AggregateMethodList: AggregateMethodList{ + { + Method: "sum", + Dimensions: []string{"__ext.container"}, + }, + { + Method: "topk", + VArgsList: []interface{}{ + 1, + }, + }, + }, + }, + }, + MetricMerge: "a", + Start: "1718865258", + End: "1718868858", + Step: "1m", + HighLight: &md.HighLight{ + Enable: true, + MaxAnalyzedOffset: 100, + }, + }, + isDirectQuery: false, + promql: `topk(1, sum by (__ext__bk_46__container) (last_over_time(a[1m])))`, + ref: md.QueryReference{ + "a": { + { + QueryList: md.QueryList{ + { + DataSource: BkLog, + Timezone: "UTC", + TableID: "result_table.es", + DataLabel: "es", + DB: "es_index", + MetricName: "usage", + VmConditionNum: 1, + VmCondition: `__name__="usage_value"`, + StorageID: "3", + StorageIDs: []string{ + "3", + }, + Field: "usage", + StorageType: consul.ElasticsearchStorageType, + Aggregates: md.Aggregates{ + { + Name: "sum", + Dimensions: []string{"__ext.container"}, + Window: time.Minute, + }, + }, + HighLight: &md.HighLight{ + Enable: true, + MaxAnalyzedOffset: 100, + }, + }, + }, + MetricName: "usage", + ReferenceName: "a", + }, + }, + }, + }, } { t.Run(name, func(t *testing.T) { var ( diff --git a/pkg/unify-query/query/structured/space.go b/pkg/unify-query/query/structured/space.go index 4cb3274e3..377a1c5f4 100644 --- a/pkg/unify-query/query/structured/space.go +++ b/pkg/unify-query/query/structured/space.go @@ -87,7 +87,7 @@ func (s *SpaceFilter) getTsDBWithResultTableDetail(t query.TsDBV2, d *routerInfl t.NeedAddTime = d.Options.NeedAddTime t.SourceType = d.SourceType - sort.SliceIsSorted(d.StorageClusterRecords, func(i, j int) bool { + sort.SliceStable(d.StorageClusterRecords, func(i, j int) bool { return d.StorageClusterRecords[i].EnableTime > d.StorageClusterRecords[j].EnableTime }) diff --git a/pkg/unify-query/query/structured/space_test.go b/pkg/unify-query/query/structured/space_test.go index df51a0373..3911bbc7a 100644 --- a/pkg/unify-query/query/structured/space_test.go +++ b/pkg/unify-query/query/structured/space_test.go @@ -43,17 +43,17 @@ func TestSpaceFilter_NewTsDBs(t *testing.T) { "test_1": { fieldName: "kube_node_info", tableID: "", - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{},"need_add_time":false,"storage_type":"influxdb"}]`, }, "test_2_regex": { fieldName: "kubelet_.+", isRegexp: true, - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{},"need_add_time":false,"storage_type":"influxdb"},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, "test_3_regex": { fieldName: "container_.+", isRegexp: true, - expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, } @@ -84,8 +84,8 @@ func TestSpaceFilter_NewTsDBs(t *testing.T) { } func toJson(q []*query.TsDBV2) string { - sort.SliceIsSorted(q, func(i, j int) bool { - return q[i].TableID < q[i].TableID + sort.SliceStable(q, func(i, j int) bool { + return q[i].TableID < q[j].TableID }) s, _ := json.Marshal(q) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index b9b9f58bc..92bc70c26 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -21,7 +21,6 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" ) // HandlerPromQLToStruct @@ -199,13 +198,12 @@ func HandlerQueryExemplar(c *gin.Context) { // @Router /query/raw [post] func HandlerQueryRaw(c *gin.Context) { var ( - ctx = c.Request.Context() - resp = &response{c: c} - user = metadata.GetUser(ctx) - err error - span *trace.Span - listData ListData - ignoreDimensions = []string{elasticsearch.KeyAddress} + ctx = c.Request.Context() + resp = &response{c: c} + user = metadata.GetUser(ctx) + err error + span *trace.Span + listData ListData ) ctx, span = trace.NewSpan(ctx, "handler-query-raw") @@ -241,7 +239,7 @@ func HandlerQueryRaw(c *gin.Context) { listData.TraceID = span.TraceID() - total, list, resultTableOptions, err := queryRawWithInstance(ctx, queryTs) + listData.Total, listData.List, listData.ResultTableOptions, err = queryRawWithInstance(ctx, queryTs) if err != nil { listData.Status = &metadata.Status{ Code: metadata.QueryRawError, @@ -249,19 +247,6 @@ func HandlerQueryRaw(c *gin.Context) { } return } - listData.Total = total - listData.List = make([]map[string]any, 0, len(list)) - for _, item := range list { - if item == nil { - continue - } - - for _, ignoreDimension := range ignoreDimensions { - delete(item, ignoreDimension) - } - listData.List = append(listData.List, item) - } - listData.ResultTableOptions = resultTableOptions resp.success(ctx, listData) } @@ -308,6 +293,11 @@ func HandlerQueryTs(c *gin.Context) { return } + // metadata 中的 spaceUid 是从 header 头信息中获取，header 如果有的话，覆盖参数里的 + if user.SpaceUid != "" { + query.SpaceUid = user.SpaceUid + } + queryStr, _ := json.Marshal(query) span.Set("query-body", string(queryStr)) span.Set("query-body-size", len(queryStr)) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 352a5ece9..7a1f5e88d 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -158,6 +158,8 @@ func queryExemplar(ctx context.Context, query *structured.QueryTs) (interface{}, } func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (total int64, list []map[string]any, resultTableOptions metadata.ResultTableOptions, err error) { + ignoreDimensions := []string{elasticsearch.KeyAddress} + ctx, span := trace.NewSpan(ctx, "query-raw-with-instance") defer span.End(&err) @@ -177,6 +179,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot lock sync.Mutex ) + list = make([]map[string]any, 0) + // 构建查询路由列表 if queryTs.SpaceUid == "" { queryTs.SpaceUid = metadata.GetUser(ctx).SpaceUid @@ -220,10 +224,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if ql.HighLight != nil { - if !ql.HighLight.Enable && queryTs.HighLight.Enable { - ql.HighLight = queryTs.HighLight - } + if ql.HighLight == nil && queryTs.HighLight != nil { + ql.HighLight = queryTs.HighLight } // 复用字段配置，没有特殊配置的情况下使用公共配置 @@ -250,14 +252,15 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot go func() { defer receiveWg.Done() + var datas []map[string]any for d := range dataCh { - list = append(list, d) + datas = append(datas, d) } span.Set("query-list-num", len(queryList)) if len(queryList) > 1 { - queryTs.OrderBy.Orders().SortSliceList(list) + queryTs.OrderBy.Orders().SortSliceList(datas) span.Set("query-scroll", queryTs.Scroll) span.Set("query-result-table", queryTs.ResultTableOptions) @@ -266,19 +269,19 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot if queryTs.Scroll == "" && queryTs.ResultTableOptions.IsCrop() { // 判定是否启用 multi from 特性 span.Set("query-multi-from", queryTs.IsMultiFrom) - span.Set("list-length", len(list)) + span.Set("datas-length", len(datas)) span.Set("query-ts-from", queryTs.From) span.Set("query-ts-limit", queryTs.Limit) - if len(list) > queryTs.Limit { + if len(datas) > queryTs.Limit { if queryTs.IsMultiFrom { resultTableOptions = queryTs.ResultTableOptions if resultTableOptions == nil { resultTableOptions = make(metadata.ResultTableOptions) } - list = list[0:queryTs.Limit] - for _, l := range list { + datas = datas[0:queryTs.Limit] + for _, l := range datas { tableID := l[elasticsearch.KeyTableID].(string) address := l[elasticsearch.KeyAddress].(string) @@ -290,10 +293,22 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } } } else { - list = list[queryTs.From : queryTs.From+queryTs.Limit] + datas = datas[queryTs.From : queryTs.From+queryTs.Limit] } } } + + } + + for _, item := range datas { + if item == nil { + continue + } + + for _, ignoreDimension := range ignoreDimensions { + delete(item, ignoreDimension) + } + list = append(list, item) } }() @@ -336,6 +351,7 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot size, options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) if queryErr != nil { message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) + return } // 如果配置了 IsMultiFrom，则无需使用 scroll 和 searchAfter 配置 @@ -362,11 +378,11 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot return } -func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs) (*PromData, error) { +func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.QueryTs) (*PromData, error) { var ( res any err error - resp = NewPromData(query.ResultColumns) + resp = NewPromData(queryTs.ResultColumns) ) ctx, span := trace.NewSpan(ctx, "query-reference") @@ -376,19 +392,36 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs span.End(&err) }() - qStr, _ := json.Marshal(query) + qStr, _ := json.Marshal(queryTs) span.Set("query-ts", string(qStr)) - for _, q := range query.QueryList { - q.IsReference = true - if q.TableID == "" { + for _, ql := range queryTs.QueryList { + ql.IsReference = true + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } + + if ql.TableID == "" { err = fmt.Errorf("tableID is empty") return nil, err } } - queryRef, err := query.ToQueryReference(ctx) - unit, startTime, endTime, err := function.QueryTimestamp(query.Start, query.End) + queryRef, err := queryTs.ToQueryReference(ctx) + unit, startTime, endTime, err := function.QueryTimestamp(queryTs.Start, queryTs.End) if err != nil { log.Errorf(ctx, err.Error()) return nil, err @@ -399,11 +432,14 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs metadata.SetQueryReference(ctx, queryRef) var lookBackDelta time.Duration - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return nil, err } + } else { + // reference 接口背后都使用了存储引擎计算，所以在不特殊指定的情况下，使用 1s 补点逻辑，防止出的数据异常 + lookBackDelta = time.Second } instance := prometheus.NewInstance(ctx, promql.GlobalEngine, &prometheus.QueryRangeStorage{ @@ -417,19 +453,19 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs ) // 只有聚合场景需要对齐 - if window, windowErr := query.GetMaxWindow(); windowErr == nil && window.Seconds() > 0 { - startTime, endTime, step, _, err = structured.AlignTime(startTime, endTime, query.Step, query.Timezone) + if window, windowErr := queryTs.GetMaxWindow(); windowErr == nil && window.Seconds() > 0 { + startTime, endTime, step, _, err = structured.AlignTime(startTime, endTime, queryTs.Step, queryTs.Timezone) if err != nil { return nil, err } } else { - step = structured.StepParse(query.Step) + step = structured.StepParse(queryTs.Step) } - if query.Instant { - res, err = instance.DirectQuery(ctx, query.MetricMerge, startTime) + if queryTs.Instant { + res, err = instance.DirectQuery(ctx, queryTs.MetricMerge, startTime) } else { - res, err = instance.DirectQueryRange(ctx, query.MetricMerge, startTime, endTime, step) + res, err = instance.DirectQueryRange(ctx, queryTs.MetricMerge, startTime, endTime, step) } if err != nil { return nil, err @@ -474,7 +510,7 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs } // queryTsToInstanceAndStmt query 结构体转换为 instance 以及 stmt -func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { +func queryTsToInstanceAndStmt(ctx context.Context, queryTs *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { var ( lookBackDelta time.Duration promExprOpt = &structured.PromExprOption{} @@ -485,37 +521,53 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i span.End(&err) }() - queryString, _ := json.Marshal(query) + queryString, _ := json.Marshal(queryTs) span.Set("query-ts", queryString) // 限制 queryList 是否过长 if DefaultQueryListLimit > 0 { - if len(query.QueryList) > DefaultQueryListLimit { + if len(queryTs.QueryList) > DefaultQueryListLimit { err = fmt.Errorf("the number of query lists cannot be greater than %d", DefaultQueryListLimit) } } // 判断是否打开对齐 - for _, q := range query.QueryList { - q.IsReference = false - q.AlignInfluxdbResult = AlignInfluxdbResult + for _, ql := range queryTs.QueryList { + ql.IsReference = false + ql.AlignInfluxdbResult = AlignInfluxdbResult + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } } // 判断是否指定 LookBackDelta - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return } } // 如果 step 为空，则补充默认 step - if query.Step == "" { - query.Step = promql.GetDefaultStep().String() + if queryTs.Step == "" { + queryTs.Step = promql.GetDefaultStep().String() } // 转换成 queryRef - queryRef, err := query.ToQueryReference(ctx) + queryRef, err := queryTs.ToQueryReference(ctx) if err != nil { return } @@ -544,7 +596,7 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i }, lookBackDelta, QueryMaxRouting) } - expr, err := query.ToPromExpr(ctx, promExprOpt) + expr, err := queryTs.ToPromExpr(ctx, promExprOpt) if err != nil { return } diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 3494408e8..c579db58a 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -17,8 +17,10 @@ import ( "time" "github.com/prometheus/prometheus/promql/parser" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/bkapi" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/featureFlag" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" @@ -35,9 +37,134 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/redis" ) +func TestQueryTsWithDoris(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + + //viper.Set(bkapi.BkAPIAddressConfigPath, mock.BkSQLUrlDomain) + + spaceUid := influxdb.SpaceUid + tableID := influxdb.ResultTableDoris + + mock.Init() + promql.MockEngine() + influxdb.MockSpaceRouter(ctx) + + defaultStart := time.UnixMilli(1744662513046) + defaultEnd := time.UnixMilli(1744684113046) + + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + // 查询 1 条原始数据，按照字段正向排序 + "SELECT *, `gseIndex` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662180000 AND `dtEventTimeStamp` <= 1744684113000 AND `thedate` = '20250415' ORDER BY `_value_` ASC LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":12,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250415,"dteventtimestamp":1744662180000,"dteventtime":null,"localtime":null,"__shard_key__":29077703953,"__ext":"{\"container_id\":\"375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2\",\"container_image\":\"sha256:3a0506f06f1467e93c3a582203aac1a7501e77091572ec9612ddeee4a4dbbdb8\",\"container_name\":\"unify-query\",\"io_kubernetes_pod\":\"bk-datalink-unify-query-6df8bcc4c9-rk4sc\",\"io_kubernetes_pod_ip\":\"127.0.0.1\",\"io_kubernetes_pod_namespace\":\"blueking\",\"io_kubernetes_pod_uid\":\"558c5b17-b221-47e1-aa66-036cc9b43e2a\",\"io_kubernetes_workload_name\":\"bk-datalink-unify-query-6df8bcc4c9\",\"io_kubernetes_workload_type\":\"ReplicaSet\"}","cloudid":0.0,"file":"http/handler.go:320","gseindex":2450131.0,"iterationindex":19.0,"level":"info","log":"2025-04-14T20:22:59.982Z\tinfo\thttp/handler.go:320\t[5108397435e997364f8dc1251533e65e] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2-json.log","report_time":"2025-04-14T20:22:59.982Z","serverip":"30.189.37.111","time":"1744662180000","trace_id":"5108397435e997364f8dc1251533e65e","_value_":2450131.0,"_timestamp_":1744662180000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":182,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":56,"match_query_routing_rule":0,"check_permission":13,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","__shard_key__","__ext","cloudid","file","gseindex","iterationindex","level","log","message","path","report_time","serverip","time","trace_id","_value_","_timestamp_"],"total_record_size":8856,"timetaken":0.255,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"long","field_name":"__c4","field_alias":"__shard_key__","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"__ext","field_index":5},{"field_type":"double","field_name":"__c6","field_alias":"cloudid","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"file","field_index":7},{"field_type":"double","field_name":"__c8","field_alias":"gseindex","field_index":8},{"field_type":"double","field_name":"__c9","field_alias":"iterationindex","field_index":9},{"field_type":"string","field_name":"__c10","field_alias":"level","field_index":10},{"field_type":"string","field_name":"__c11","field_alias":"log","field_index":11},{"field_type":"string","field_name":"__c12","field_alias":"message","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"path","field_index":13},{"field_type":"string","field_name":"__c14","field_alias":"report_time","field_index":14},{"field_type":"string","field_name":"__c15","field_alias":"serverip","field_index":15},{"field_type":"string","field_name":"__c16","field_alias":"time","field_index":16},{"field_type":"string","field_name":"__c17","field_alias":"trace_id","field_index":17},{"field_type":"double","field_name":"__c18","field_alias":"_value_","field_index":18},{"field_type":"long","field_name":"__c19","field_alias":"_timestamp_","field_index":19}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + // 根据维度 __ext.container_name 进行 count 聚合，同时用值正向排序 + "SELECT CAST(__ext[\"container_name\"] AS STRING) AS `__ext__bk_46__container_name`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 30000 AS INT) * 30000 AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662509999 AND `dtEventTimeStamp` <= 1744684142999 AND `thedate` = '20250415' GROUP BY __ext__bk_46__container_name, _timestamp_ ORDER BY `_timestamp_` ASC, `_value_` ASC LIMIT 2000005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":722,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":6,"bkbase_apigw_api":28},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"__ext__bk_46__container_name":"unify-query","_value_":3684,"_timestamp_":1744662510000},{"__ext__bk_46__container_name":"unify-query","_value_":4012,"_timestamp_":1744662540000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744662570000},{"__ext__bk_46__container_name":"unify-query","_value_":17092,"_timestamp_":1744662600000},{"__ext__bk_46__container_name":"unify-query","_value_":12881,"_timestamp_":1744662630000},{"__ext__bk_46__container_name":"unify-query","_value_":5902,"_timestamp_":1744662660000},{"__ext__bk_46__container_name":"unify-query","_value_":10443,"_timestamp_":1744662690000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744662720000},{"__ext__bk_46__container_name":"unify-query","_value_":3357,"_timestamp_":1744662750000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744662780000},{"__ext__bk_46__container_name":"unify-query","_value_":3683,"_timestamp_":1744662810000},{"__ext__bk_46__container_name":"unify-query","_value_":4353,"_timestamp_":1744662840000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744662870000},{"__ext__bk_46__container_name":"unify-query","_value_":4251,"_timestamp_":1744662900000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744662930000},{"__ext__bk_46__container_name":"unify-query","_value_":4036,"_timestamp_":1744662960000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744662990000},{"__ext__bk_46__container_name":"unify-query","_value_":4351,"_timestamp_":1744663020000},{"__ext__bk_46__container_name":"unify-query","_value_":3651,"_timestamp_":1744663050000},{"__ext__bk_46__container_name":"unify-query","_value_":4096,"_timestamp_":1744663080000},{"__ext__bk_46__container_name":"unify-query","_value_":3618,"_timestamp_":1744663110000},{"__ext__bk_46__container_name":"unify-query","_value_":4100,"_timestamp_":1744663140000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744663170000},{"__ext__bk_46__container_name":"unify-query","_value_":6044,"_timestamp_":1744663200000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744663230000},{"__ext__bk_46__container_name":"unify-query","_value_":4461,"_timestamp_":1744663260000},{"__ext__bk_46__container_name":"unify-query","_value_":3783,"_timestamp_":1744663290000},{"__ext__bk_46__container_name":"unify-query","_value_":4559,"_timestamp_":1744663320000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744663350000},{"__ext__bk_46__container_name":"unify-query","_value_":3869,"_timestamp_":1744663380000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744663410000},{"__ext__bk_46__container_name":"unify-query","_value_":4473,"_timestamp_":1744663440000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744663470000},{"__ext__bk_46__container_name":"unify-query","_value_":4923,"_timestamp_":1744663500000},{"__ext__bk_46__container_name":"unify-query","_value_":3379,"_timestamp_":1744663530000},{"__ext__bk_46__container_name":"unify-query","_value_":4489,"_timestamp_":1744663560000},{"__ext__bk_46__container_name":"unify-query","_value_":3411,"_timestamp_":1744663590000},{"__ext__bk_46__container_name":"unify-query","_value_":4374,"_timestamp_":1744663620000},{"__ext__bk_46__container_name":"unify-query","_value_":3370,"_timestamp_":1744663650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744663680000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744663710000},{"__ext__bk_46__container_name":"unify-query","_value_":4318,"_timestamp_":1744663740000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744663770000},{"__ext__bk_46__container_name":"unify-query","_value_":4334,"_timestamp_":1744663800000},{"__ext__bk_46__container_name":"unify-query","_value_":3767,"_timestamp_":1744663830000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744663860000},{"__ext__bk_46__container_name":"unify-query","_value_":3703,"_timestamp_":1744663890000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744663920000},{"__ext__bk_46__container_name":"unify-query","_value_":3667,"_timestamp_":1744663950000},{"__ext__bk_46__container_name":"unify-query","_value_":3998,"_timestamp_":1744663980000},{"__ext__bk_46__container_name":"unify-query","_value_":3579,"_timestamp_":1744664010000},{"__ext__bk_46__container_name":"unify-query","_value_":4156,"_timestamp_":1744664040000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744664070000},{"__ext__bk_46__container_name":"unify-query","_value_":4344,"_timestamp_":1744664100000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744664130000},{"__ext__bk_46__container_name":"unify-query","_value_":4161,"_timestamp_":1744664160000},{"__ext__bk_46__container_name":"unify-query","_value_":3484,"_timestamp_":1744664190000},{"__ext__bk_46__container_name":"unify-query","_value_":4273,"_timestamp_":1744664220000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744664250000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744664280000},{"__ext__bk_46__container_name":"unify-query","_value_":3619,"_timestamp_":1744664310000},{"__ext__bk_46__container_name":"unify-query","_value_":4013,"_timestamp_":1744664340000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744664370000},{"__ext__bk_46__container_name":"unify-query","_value_":18144,"_timestamp_":1744664400000},{"__ext__bk_46__container_name":"unify-query","_value_":13615,"_timestamp_":1744664430000},{"__ext__bk_46__container_name":"unify-query","_value_":3178,"_timestamp_":1744664460000},{"__ext__bk_46__container_name":"unify-query","_value_":13044,"_timestamp_":1744664490000},{"__ext__bk_46__container_name":"unify-query","_value_":4767,"_timestamp_":1744664520000},{"__ext__bk_46__container_name":"unify-query","_value_":3528,"_timestamp_":1744664550000},{"__ext__bk_46__container_name":"unify-query","_value_":4316,"_timestamp_":1744664580000},{"__ext__bk_46__container_name":"unify-query","_value_":3317,"_timestamp_":1744664610000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744664640000},{"__ext__bk_46__container_name":"unify-query","_value_":3599,"_timestamp_":1744664670000},{"__ext__bk_46__container_name":"unify-query","_value_":4149,"_timestamp_":1744664700000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744664730000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744664760000},{"__ext__bk_46__container_name":"unify-query","_value_":3384,"_timestamp_":1744664790000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744664820000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744664850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744664880000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744664910000},{"__ext__bk_46__container_name":"unify-query","_value_":4244,"_timestamp_":1744664940000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744664970000},{"__ext__bk_46__container_name":"unify-query","_value_":4305,"_timestamp_":1744665000000},{"__ext__bk_46__container_name":"unify-query","_value_":3771,"_timestamp_":1744665030000},{"__ext__bk_46__container_name":"unify-query","_value_":4485,"_timestamp_":1744665060000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744665090000},{"__ext__bk_46__container_name":"unify-query","_value_":4423,"_timestamp_":1744665120000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665150000},{"__ext__bk_46__container_name":"unify-query","_value_":4125,"_timestamp_":1744665180000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744665210000},{"__ext__bk_46__container_name":"unify-query","_value_":4252,"_timestamp_":1744665240000},{"__ext__bk_46__container_name":"unify-query","_value_":3427,"_timestamp_":1744665270000},{"__ext__bk_46__container_name":"unify-query","_value_":5089,"_timestamp_":1744665300000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744665330000},{"__ext__bk_46__container_name":"unify-query","_value_":4349,"_timestamp_":1744665360000},{"__ext__bk_46__container_name":"unify-query","_value_":3188,"_timestamp_":1744665390000},{"__ext__bk_46__container_name":"unify-query","_value_":4556,"_timestamp_":1744665420000},{"__ext__bk_46__container_name":"unify-query","_value_":3372,"_timestamp_":1744665450000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744665480000},{"__ext__bk_46__container_name":"unify-query","_value_":3445,"_timestamp_":1744665510000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744665540000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744665570000},{"__ext__bk_46__container_name":"unify-query","_value_":6235,"_timestamp_":1744665600000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744665630000},{"__ext__bk_46__container_name":"unify-query","_value_":4577,"_timestamp_":1744665660000},{"__ext__bk_46__container_name":"unify-query","_value_":3719,"_timestamp_":1744665690000},{"__ext__bk_46__container_name":"unify-query","_value_":4548,"_timestamp_":1744665720000},{"__ext__bk_46__container_name":"unify-query","_value_":3420,"_timestamp_":1744665750000},{"__ext__bk_46__container_name":"unify-query","_value_":4246,"_timestamp_":1744665780000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744665810000},{"__ext__bk_46__container_name":"unify-query","_value_":4332,"_timestamp_":1744665840000},{"__ext__bk_46__container_name":"unify-query","_value_":3422,"_timestamp_":1744665870000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744665900000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665930000},{"__ext__bk_46__container_name":"unify-query","_value_":4119,"_timestamp_":1744665960000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744665990000},{"__ext__bk_46__container_name":"unify-query","_value_":4144,"_timestamp_":1744666020000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744666050000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744666080000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744666110000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744666140000},{"__ext__bk_46__container_name":"unify-query","_value_":3315,"_timestamp_":1744666170000},{"__ext__bk_46__container_name":"unify-query","_value_":16721,"_timestamp_":1744666200000},{"__ext__bk_46__container_name":"unify-query","_value_":13631,"_timestamp_":1744666230000},{"__ext__bk_46__container_name":"unify-query","_value_":2982,"_timestamp_":1744666260000},{"__ext__bk_46__container_name":"unify-query","_value_":11858,"_timestamp_":1744666290000},{"__ext__bk_46__container_name":"unify-query","_value_":5515,"_timestamp_":1744666320000},{"__ext__bk_46__container_name":"unify-query","_value_":2869,"_timestamp_":1744666350000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744666380000},{"__ext__bk_46__container_name":"unify-query","_value_":3603,"_timestamp_":1744666410000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744666440000},{"__ext__bk_46__container_name":"unify-query","_value_":3264,"_timestamp_":1744666470000},{"__ext__bk_46__container_name":"unify-query","_value_":4377,"_timestamp_":1744666500000},{"__ext__bk_46__container_name":"unify-query","_value_":3443,"_timestamp_":1744666530000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744666560000},{"__ext__bk_46__container_name":"unify-query","_value_":3459,"_timestamp_":1744666590000},{"__ext__bk_46__container_name":"unify-query","_value_":4342,"_timestamp_":1744666620000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744666650000},{"__ext__bk_46__container_name":"unify-query","_value_":4052,"_timestamp_":1744666680000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744666710000},{"__ext__bk_46__container_name":"unify-query","_value_":4128,"_timestamp_":1744666740000},{"__ext__bk_46__container_name":"unify-query","_value_":3499,"_timestamp_":1744666770000},{"__ext__bk_46__container_name":"unify-query","_value_":6209,"_timestamp_":1744666800000},{"__ext__bk_46__container_name":"unify-query","_value_":3575,"_timestamp_":1744666830000},{"__ext__bk_46__container_name":"unify-query","_value_":4543,"_timestamp_":1744666860000},{"__ext__bk_46__container_name":"unify-query","_value_":3604,"_timestamp_":1744666890000},{"__ext__bk_46__container_name":"unify-query","_value_":4579,"_timestamp_":1744666920000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744666950000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744666980000},{"__ext__bk_46__container_name":"unify-query","_value_":3416,"_timestamp_":1744667010000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667040000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744667070000},{"__ext__bk_46__container_name":"unify-query","_value_":5054,"_timestamp_":1744667100000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744667130000},{"__ext__bk_46__container_name":"unify-query","_value_":4313,"_timestamp_":1744667160000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744667190000},{"__ext__bk_46__container_name":"unify-query","_value_":4118,"_timestamp_":1744667220000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744667250000},{"__ext__bk_46__container_name":"unify-query","_value_":4280,"_timestamp_":1744667280000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744667310000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744667340000},{"__ext__bk_46__container_name":"unify-query","_value_":3807,"_timestamp_":1744667370000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744667400000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744667430000},{"__ext__bk_46__container_name":"unify-query","_value_":4630,"_timestamp_":1744667460000},{"__ext__bk_46__container_name":"unify-query","_value_":3845,"_timestamp_":1744667490000},{"__ext__bk_46__container_name":"unify-query","_value_":4361,"_timestamp_":1744667520000},{"__ext__bk_46__container_name":"unify-query","_value_":3572,"_timestamp_":1744667550000},{"__ext__bk_46__container_name":"unify-query","_value_":4095,"_timestamp_":1744667580000},{"__ext__bk_46__container_name":"unify-query","_value_":3535,"_timestamp_":1744667610000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744667640000},{"__ext__bk_46__container_name":"unify-query","_value_":3390,"_timestamp_":1744667670000},{"__ext__bk_46__container_name":"unify-query","_value_":4262,"_timestamp_":1744667700000},{"__ext__bk_46__container_name":"unify-query","_value_":3398,"_timestamp_":1744667730000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667760000},{"__ext__bk_46__container_name":"unify-query","_value_":3429,"_timestamp_":1744667790000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744667820000},{"__ext__bk_46__container_name":"unify-query","_value_":3482,"_timestamp_":1744667850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744667880000},{"__ext__bk_46__container_name":"unify-query","_value_":3612,"_timestamp_":1744667910000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744667940000},{"__ext__bk_46__container_name":"unify-query","_value_":3423,"_timestamp_":1744667970000},{"__ext__bk_46__container_name":"unify-query","_value_":18203,"_timestamp_":1744668000000},{"__ext__bk_46__container_name":"unify-query","_value_":13685,"_timestamp_":1744668030000},{"__ext__bk_46__container_name":"unify-query","_value_":3281,"_timestamp_":1744668060000},{"__ext__bk_46__container_name":"unify-query","_value_":12556,"_timestamp_":1744668090000},{"__ext__bk_46__container_name":"unify-query","_value_":4893,"_timestamp_":1744668120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744668150000},{"__ext__bk_46__container_name":"unify-query","_value_":4336,"_timestamp_":1744668180000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744668210000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744668240000},{"__ext__bk_46__container_name":"unify-query","_value_":3669,"_timestamp_":1744668270000},{"__ext__bk_46__container_name":"unify-query","_value_":3997,"_timestamp_":1744668300000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744668330000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744668360000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744668390000},{"__ext__bk_46__container_name":"unify-query","_value_":3877,"_timestamp_":1744668420000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744668450000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744668480000},{"__ext__bk_46__container_name":"unify-query","_value_":3469,"_timestamp_":1744668510000},{"__ext__bk_46__container_name":"unify-query","_value_":4243,"_timestamp_":1744668540000},{"__ext__bk_46__container_name":"unify-query","_value_":3304,"_timestamp_":1744668570000},{"__ext__bk_46__container_name":"unify-query","_value_":4690,"_timestamp_":1744668600000},{"__ext__bk_46__container_name":"unify-query","_value_":3717,"_timestamp_":1744668630000},{"__ext__bk_46__container_name":"unify-query","_value_":4618,"_timestamp_":1744668660000},{"__ext__bk_46__container_name":"unify-query","_value_":3732,"_timestamp_":1744668690000},{"__ext__bk_46__container_name":"unify-query","_value_":4477,"_timestamp_":1744668720000},{"__ext__bk_46__container_name":"unify-query","_value_":3615,"_timestamp_":1744668750000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744668780000},{"__ext__bk_46__container_name":"unify-query","_value_":3367,"_timestamp_":1744668810000},{"__ext__bk_46__container_name":"unify-query","_value_":4193,"_timestamp_":1744668840000},{"__ext__bk_46__container_name":"unify-query","_value_":3592,"_timestamp_":1744668870000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744668900000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744668930000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744668960000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744668990000},{"__ext__bk_46__container_name":"unify-query","_value_":4375,"_timestamp_":1744669020000},{"__ext__bk_46__container_name":"unify-query","_value_":3386,"_timestamp_":1744669050000},{"__ext__bk_46__container_name":"unify-query","_value_":4281,"_timestamp_":1744669080000},{"__ext__bk_46__container_name":"unify-query","_value_":3410,"_timestamp_":1744669110000},{"__ext__bk_46__container_name":"unify-query","_value_":4545,"_timestamp_":1744669140000},{"__ext__bk_46__container_name":"unify-query","_value_":3724,"_timestamp_":1744669170000},{"__ext__bk_46__container_name":"unify-query","_value_":5903,"_timestamp_":1744669200000},{"__ext__bk_46__container_name":"unify-query","_value_":3672,"_timestamp_":1744669230000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744669260000},{"__ext__bk_46__container_name":"unify-query","_value_":3792,"_timestamp_":1744669290000},{"__ext__bk_46__container_name":"unify-query","_value_":4422,"_timestamp_":1744669320000},{"__ext__bk_46__container_name":"unify-query","_value_":3718,"_timestamp_":1744669350000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744669380000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744669410000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744669440000},{"__ext__bk_46__container_name":"unify-query","_value_":3542,"_timestamp_":1744669470000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744669500000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669530000},{"__ext__bk_46__container_name":"unify-query","_value_":4354,"_timestamp_":1744669560000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669590000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744669620000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744669650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744669680000},{"__ext__bk_46__container_name":"unify-query","_value_":3512,"_timestamp_":1744669710000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744669740000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744669770000},{"__ext__bk_46__container_name":"unify-query","_value_":12171,"_timestamp_":1744669800000},{"__ext__bk_46__container_name":"unify-query","_value_":18129,"_timestamp_":1744669830000},{"__ext__bk_46__container_name":"unify-query","_value_":7142,"_timestamp_":1744669860000},{"__ext__bk_46__container_name":"unify-query","_value_":9153,"_timestamp_":1744669890000},{"__ext__bk_46__container_name":"unify-query","_value_":4566,"_timestamp_":1744669920000},{"__ext__bk_46__container_name":"unify-query","_value_":3225,"_timestamp_":1744669950000},{"__ext__bk_46__container_name":"unify-query","_value_":4378,"_timestamp_":1744669980000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744670010000},{"__ext__bk_46__container_name":"unify-query","_value_":4266,"_timestamp_":1744670040000},{"__ext__bk_46__container_name":"unify-query","_value_":3645,"_timestamp_":1744670070000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744670100000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744670130000},{"__ext__bk_46__container_name":"unify-query","_value_":4333,"_timestamp_":1744670160000},{"__ext__bk_46__container_name":"unify-query","_value_":3489,"_timestamp_":1744670190000},{"__ext__bk_46__container_name":"unify-query","_value_":4303,"_timestamp_":1744670220000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744670250000},{"__ext__bk_46__container_name":"unify-query","_value_":4121,"_timestamp_":1744670280000},{"__ext__bk_46__container_name":"unify-query","_value_":3374,"_timestamp_":1744670310000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744670340000},{"__ext__bk_46__container_name":"unify-query","_value_":3242,"_timestamp_":1744670370000},{"__ext__bk_46__container_name":"unify-query","_value_":6416,"_timestamp_":1744670400000},{"__ext__bk_46__container_name":"unify-query","_value_":3697,"_timestamp_":1744670430000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744670460000},{"__ext__bk_46__container_name":"unify-query","_value_":3749,"_timestamp_":1744670490000},{"__ext__bk_46__container_name":"unify-query","_value_":4587,"_timestamp_":1744670520000},{"__ext__bk_46__container_name":"unify-query","_value_":3538,"_timestamp_":1744670550000},{"__ext__bk_46__container_name":"unify-query","_value_":4221,"_timestamp_":1744670580000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744670610000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744670640000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744670670000},{"__ext__bk_46__container_name":"unify-query","_value_":4848,"_timestamp_":1744670700000},{"__ext__bk_46__container_name":"unify-query","_value_":3551,"_timestamp_":1744670730000},{"__ext__bk_46__container_name":"unify-query","_value_":4068,"_timestamp_":1744670760000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744670790000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744670820000},{"__ext__bk_46__container_name":"unify-query","_value_":3635,"_timestamp_":1744670850000},{"__ext__bk_46__container_name":"unify-query","_value_":4256,"_timestamp_":1744670880000},{"__ext__bk_46__container_name":"unify-query","_value_":3690,"_timestamp_":1744670910000},{"__ext__bk_46__container_name":"unify-query","_value_":4155,"_timestamp_":1744670940000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744670970000},{"__ext__bk_46__container_name":"unify-query","_value_":4661,"_timestamp_":1744671000000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744671030000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744671060000},{"__ext__bk_46__container_name":"unify-query","_value_":3643,"_timestamp_":1744671090000},{"__ext__bk_46__container_name":"unify-query","_value_":4755,"_timestamp_":1744671120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744671150000},{"__ext__bk_46__container_name":"unify-query","_value_":4284,"_timestamp_":1744671180000},{"__ext__bk_46__container_name":"unify-query","_value_":3258,"_timestamp_":1744671210000},{"__ext__bk_46__container_name":"unify-query","_value_":4453,"_timestamp_":1744671240000},{"__ext__bk_46__container_name":"unify-query","_value_":3431,"_timestamp_":1744671270000},{"__ext__bk_46__container_name":"unify-query","_value_":4231,"_timestamp_":1744671300000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744671330000},{"__ext__bk_46__container_name":"unify-query","_value_":3907,"_timestamp_":1744671360000},{"__ext__bk_46__container_name":"unify-query","_value_":3524,"_timestamp_":1744671390000},{"__ext__bk_46__container_name":"unify-query","_value_":4438,"_timestamp_":1744671420000},{"__ext__bk_46__container_name":"unify-query","_value_":3547,"_timestamp_":1744671450000},{"__ext__bk_46__container_name":"unify-query","_value_":4033,"_timestamp_":1744671480000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744671510000},{"__ext__bk_46__container_name":"unify-query","_value_":4162,"_timestamp_":1744671540000},{"__ext__bk_46__container_name":"unify-query","_value_":3588,"_timestamp_":1744671570000},{"__ext__bk_46__container_name":"unify-query","_value_":16444,"_timestamp_":1744671600000},{"__ext__bk_46__container_name":"unify-query","_value_":15396,"_timestamp_":1744671630000},{"__ext__bk_46__container_name":"unify-query","_value_":3024,"_timestamp_":1744671660000},{"__ext__bk_46__container_name":"unify-query","_value_":12656,"_timestamp_":1744671690000},{"__ext__bk_46__container_name":"unify-query","_value_":4733,"_timestamp_":1744671720000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744671750000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744671780000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744671810000},{"__ext__bk_46__container_name":"unify-query","_value_":4487,"_timestamp_":1744671840000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744671870000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744671900000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744671930000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744671960000},{"__ext__bk_46__container_name":"unify-query","_value_":3472,"_timestamp_":1744671990000},{"__ext__bk_46__container_name":"unify-query","_value_":4309,"_timestamp_":1744672020000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744672050000},{"__ext__bk_46__container_name":"unify-query","_value_":4191,"_timestamp_":1744672080000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744672110000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744672140000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744672170000},{"__ext__bk_46__container_name":"unify-query","_value_":4445,"_timestamp_":1744672200000},{"__ext__bk_46__container_name":"unify-query","_value_":3822,"_timestamp_":1744672230000},{"__ext__bk_46__container_name":"unify-query","_value_":4346,"_timestamp_":1744672260000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744672290000},{"__ext__bk_46__container_name":"unify-query","_value_":4615,"_timestamp_":1744672320000},{"__ext__bk_46__container_name":"unify-query","_value_":3591,"_timestamp_":1744672350000},{"__ext__bk_46__container_name":"unify-query","_value_":4056,"_timestamp_":1744672380000},{"__ext__bk_46__container_name":"unify-query","_value_":3544,"_timestamp_":1744672410000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744672440000},{"__ext__bk_46__container_name":"unify-query","_value_":3647,"_timestamp_":1744672470000},{"__ext__bk_46__container_name":"unify-query","_value_":4887,"_timestamp_":1744672500000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744672530000},{"__ext__bk_46__container_name":"unify-query","_value_":4302,"_timestamp_":1744672560000},{"__ext__bk_46__container_name":"unify-query","_value_":3425,"_timestamp_":1744672590000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744672620000},{"__ext__bk_46__container_name":"unify-query","_value_":3532,"_timestamp_":1744672650000},{"__ext__bk_46__container_name":"unify-query","_value_":4282,"_timestamp_":1744672680000},{"__ext__bk_46__container_name":"unify-query","_value_":3571,"_timestamp_":1744672710000},{"__ext__bk_46__container_name":"unify-query","_value_":4182,"_timestamp_":1744672740000},{"__ext__bk_46__container_name":"unify-query","_value_":3210,"_timestamp_":1744672770000},{"__ext__bk_46__container_name":"unify-query","_value_":6383,"_timestamp_":1744672800000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744672830000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744672860000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744672890000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744672920000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744672950000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744672980000},{"__ext__bk_46__container_name":"unify-query","_value_":3534,"_timestamp_":1744673010000},{"__ext__bk_46__container_name":"unify-query","_value_":4212,"_timestamp_":1744673040000},{"__ext__bk_46__container_name":"unify-query","_value_":3380,"_timestamp_":1744673070000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744673100000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744673130000},{"__ext__bk_46__container_name":"unify-query","_value_":4120,"_timestamp_":1744673160000},{"__ext__bk_46__container_name":"unify-query","_value_":3526,"_timestamp_":1744673190000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744673220000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744673250000},{"__ext__bk_46__container_name":"unify-query","_value_":4370,"_timestamp_":1744673280000},{"__ext__bk_46__container_name":"unify-query","_value_":3462,"_timestamp_":1744673310000},{"__ext__bk_46__container_name":"unify-query","_value_":4223,"_timestamp_":1744673340000},{"__ext__bk_46__container_name":"unify-query","_value_":3564,"_timestamp_":1744673370000},{"__ext__bk_46__container_name":"unify-query","_value_":12072,"_timestamp_":1744673400000},{"__ext__bk_46__container_name":"unify-query","_value_":17986,"_timestamp_":1744673430000},{"__ext__bk_46__container_name":"unify-query","_value_":4089,"_timestamp_":1744673460000},{"__ext__bk_46__container_name":"unify-query","_value_":12000,"_timestamp_":1744673490000},{"__ext__bk_46__container_name":"unify-query","_value_":4790,"_timestamp_":1744673520000},{"__ext__bk_46__container_name":"unify-query","_value_":3637,"_timestamp_":1744673550000},{"__ext__bk_46__container_name":"unify-query","_value_":4177,"_timestamp_":1744673580000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744673610000},{"__ext__bk_46__container_name":"unify-query","_value_":4465,"_timestamp_":1744673640000},{"__ext__bk_46__container_name":"unify-query","_value_":3627,"_timestamp_":1744673670000},{"__ext__bk_46__container_name":"unify-query","_value_":4131,"_timestamp_":1744673700000},{"__ext__bk_46__container_name":"unify-query","_value_":3396,"_timestamp_":1744673730000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744673760000},{"__ext__bk_46__container_name":"unify-query","_value_":3638,"_timestamp_":1744673790000},{"__ext__bk_46__container_name":"unify-query","_value_":4093,"_timestamp_":1744673820000},{"__ext__bk_46__container_name":"unify-query","_value_":3584,"_timestamp_":1744673850000},{"__ext__bk_46__container_name":"unify-query","_value_":4082,"_timestamp_":1744673880000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744673910000},{"__ext__bk_46__container_name":"unify-query","_value_":4051,"_timestamp_":1744673940000},{"__ext__bk_46__container_name":"unify-query","_value_":3354,"_timestamp_":1744673970000},{"__ext__bk_46__container_name":"unify-query","_value_":6296,"_timestamp_":1744674000000},{"__ext__bk_46__container_name":"unify-query","_value_":3473,"_timestamp_":1744674030000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744674060000},{"__ext__bk_46__container_name":"unify-query","_value_":3793,"_timestamp_":1744674090000},{"__ext__bk_46__container_name":"unify-query","_value_":4391,"_timestamp_":1744674120000},{"__ext__bk_46__container_name":"unify-query","_value_":3836,"_timestamp_":1744674150000},{"__ext__bk_46__container_name":"unify-query","_value_":4190,"_timestamp_":1744674180000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674210000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744674240000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744674270000},{"__ext__bk_46__container_name":"unify-query","_value_":4964,"_timestamp_":1744674300000},{"__ext__bk_46__container_name":"unify-query","_value_":3455,"_timestamp_":1744674330000},{"__ext__bk_46__container_name":"unify-query","_value_":4116,"_timestamp_":1744674360000},{"__ext__bk_46__container_name":"unify-query","_value_":3250,"_timestamp_":1744674390000},{"__ext__bk_46__container_name":"unify-query","_value_":4494,"_timestamp_":1744674420000},{"__ext__bk_46__container_name":"unify-query","_value_":3326,"_timestamp_":1744674450000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744674480000},{"__ext__bk_46__container_name":"unify-query","_value_":3580,"_timestamp_":1744674510000},{"__ext__bk_46__container_name":"unify-query","_value_":4368,"_timestamp_":1744674540000},{"__ext__bk_46__container_name":"unify-query","_value_":3685,"_timestamp_":1744674570000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744674600000},{"__ext__bk_46__container_name":"unify-query","_value_":3699,"_timestamp_":1744674630000},{"__ext__bk_46__container_name":"unify-query","_value_":4513,"_timestamp_":1744674660000},{"__ext__bk_46__container_name":"unify-query","_value_":3729,"_timestamp_":1744674690000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744674720000},{"__ext__bk_46__container_name":"unify-query","_value_":3639,"_timestamp_":1744674750000},{"__ext__bk_46__container_name":"unify-query","_value_":4018,"_timestamp_":1744674780000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744674810000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744674840000},{"__ext__bk_46__container_name":"unify-query","_value_":3389,"_timestamp_":1744674870000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744674900000},{"__ext__bk_46__container_name":"unify-query","_value_":3540,"_timestamp_":1744674930000},{"__ext__bk_46__container_name":"unify-query","_value_":4106,"_timestamp_":1744674960000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674990000},{"__ext__bk_46__container_name":"unify-query","_value_":4268,"_timestamp_":1744675020000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744675050000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744675080000},{"__ext__bk_46__container_name":"unify-query","_value_":3511,"_timestamp_":1744675110000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744675140000},{"__ext__bk_46__container_name":"unify-query","_value_":3573,"_timestamp_":1744675170000},{"__ext__bk_46__container_name":"unify-query","_value_":17095,"_timestamp_":1744675200000},{"__ext__bk_46__container_name":"unify-query","_value_":14907,"_timestamp_":1744675230000},{"__ext__bk_46__container_name":"unify-query","_value_":6455,"_timestamp_":1744675260000},{"__ext__bk_46__container_name":"unify-query","_value_":9818,"_timestamp_":1744675290000},{"__ext__bk_46__container_name":"unify-query","_value_":5253,"_timestamp_":1744675320000},{"__ext__bk_46__container_name":"unify-query","_value_":3567,"_timestamp_":1744675350000},{"__ext__bk_46__container_name":"unify-query","_value_":4047,"_timestamp_":1744675380000},{"__ext__bk_46__container_name":"unify-query","_value_":3342,"_timestamp_":1744675410000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744675440000},{"__ext__bk_46__container_name":"unify-query","_value_":3394,"_timestamp_":1744675470000},{"__ext__bk_46__container_name":"unify-query","_value_":4260,"_timestamp_":1744675500000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744675530000},{"__ext__bk_46__container_name":"unify-query","_value_":4341,"_timestamp_":1744675560000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744675590000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744675620000},{"__ext__bk_46__container_name":"unify-query","_value_":3519,"_timestamp_":1744675650000},{"__ext__bk_46__container_name":"unify-query","_value_":4143,"_timestamp_":1744675680000},{"__ext__bk_46__container_name":"unify-query","_value_":3630,"_timestamp_":1744675710000},{"__ext__bk_46__container_name":"unify-query","_value_":4042,"_timestamp_":1744675740000},{"__ext__bk_46__container_name":"unify-query","_value_":3653,"_timestamp_":1744675770000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744675800000},{"__ext__bk_46__container_name":"unify-query","_value_":3688,"_timestamp_":1744675830000},{"__ext__bk_46__container_name":"unify-query","_value_":4450,"_timestamp_":1744675860000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744675890000},{"__ext__bk_46__container_name":"unify-query","_value_":4864,"_timestamp_":1744675920000},{"__ext__bk_46__container_name":"unify-query","_value_":3629,"_timestamp_":1744675950000},{"__ext__bk_46__container_name":"unify-query","_value_":4127,"_timestamp_":1744675980000},{"__ext__bk_46__container_name":"unify-query","_value_":3424,"_timestamp_":1744676010000},{"__ext__bk_46__container_name":"unify-query","_value_":4267,"_timestamp_":1744676040000},{"__ext__bk_46__container_name":"unify-query","_value_":3328,"_timestamp_":1744676070000},{"__ext__bk_46__container_name":"unify-query","_value_":5128,"_timestamp_":1744676100000},{"__ext__bk_46__container_name":"unify-query","_value_":3657,"_timestamp_":1744676130000},{"__ext__bk_46__container_name":"unify-query","_value_":4185,"_timestamp_":1744676160000},{"__ext__bk_46__container_name":"unify-query","_value_":3336,"_timestamp_":1744676190000},{"__ext__bk_46__container_name":"unify-query","_value_":4532,"_timestamp_":1744676220000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744676250000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744676280000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744676310000},{"__ext__bk_46__container_name":"unify-query","_value_":4463,"_timestamp_":1744676340000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676370000},{"__ext__bk_46__container_name":"unify-query","_value_":6064,"_timestamp_":1744676400000},{"__ext__bk_46__container_name":"unify-query","_value_":3292,"_timestamp_":1744676430000},{"__ext__bk_46__container_name":"unify-query","_value_":4858,"_timestamp_":1744676460000},{"__ext__bk_46__container_name":"unify-query","_value_":3543,"_timestamp_":1744676490000},{"__ext__bk_46__container_name":"unify-query","_value_":4620,"_timestamp_":1744676520000},{"__ext__bk_46__container_name":"unify-query","_value_":3750,"_timestamp_":1744676550000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744676580000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744676610000},{"__ext__bk_46__container_name":"unify-query","_value_":4152,"_timestamp_":1744676640000},{"__ext__bk_46__container_name":"unify-query","_value_":3550,"_timestamp_":1744676670000},{"__ext__bk_46__container_name":"unify-query","_value_":4011,"_timestamp_":1744676700000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676730000},{"__ext__bk_46__container_name":"unify-query","_value_":4050,"_timestamp_":1744676760000},{"__ext__bk_46__container_name":"unify-query","_value_":3118,"_timestamp_":1744676790000},{"__ext__bk_46__container_name":"unify-query","_value_":4628,"_timestamp_":1744676820000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744676850000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744676880000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744676910000},{"__ext__bk_46__container_name":"unify-query","_value_":4160,"_timestamp_":1744676940000},{"__ext__bk_46__container_name":"unify-query","_value_":3662,"_timestamp_":1744676970000},{"__ext__bk_46__container_name":"unify-query","_value_":11392,"_timestamp_":1744677000000},{"__ext__bk_46__container_name":"unify-query","_value_":18649,"_timestamp_":1744677030000},{"__ext__bk_46__container_name":"unify-query","_value_":7107,"_timestamp_":1744677060000},{"__ext__bk_46__container_name":"unify-query","_value_":9213,"_timestamp_":1744677090000},{"__ext__bk_46__container_name":"unify-query","_value_":4235,"_timestamp_":1744677120000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744677150000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744677180000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744677210000},{"__ext__bk_46__container_name":"unify-query","_value_":4233,"_timestamp_":1744677240000},{"__ext__bk_46__container_name":"unify-query","_value_":3440,"_timestamp_":1744677270000},{"__ext__bk_46__container_name":"unify-query","_value_":4383,"_timestamp_":1744677300000},{"__ext__bk_46__container_name":"unify-query","_value_":3507,"_timestamp_":1744677330000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744677360000},{"__ext__bk_46__container_name":"unify-query","_value_":3197,"_timestamp_":1744677390000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744677420000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744677450000},{"__ext__bk_46__container_name":"unify-query","_value_":4421,"_timestamp_":1744677480000},{"__ext__bk_46__container_name":"unify-query","_value_":2998,"_timestamp_":1744677510000},{"__ext__bk_46__container_name":"unify-query","_value_":4700,"_timestamp_":1744677540000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744677570000},{"__ext__bk_46__container_name":"unify-query","_value_":5781,"_timestamp_":1744677600000},{"__ext__bk_46__container_name":"unify-query","_value_":3734,"_timestamp_":1744677630000},{"__ext__bk_46__container_name":"unify-query","_value_":4510,"_timestamp_":1744677660000},{"__ext__bk_46__container_name":"unify-query","_value_":3752,"_timestamp_":1744677690000},{"__ext__bk_46__container_name":"unify-query","_value_":4447,"_timestamp_":1744677720000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744677750000},{"__ext__bk_46__container_name":"unify-query","_value_":4187,"_timestamp_":1744677780000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744677810000},{"__ext__bk_46__container_name":"unify-query","_value_":3900,"_timestamp_":1744677840000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744677870000},{"__ext__bk_46__container_name":"unify-query","_value_":4863,"_timestamp_":1744677900000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744677930000},{"__ext__bk_46__container_name":"unify-query","_value_":4335,"_timestamp_":1744677960000},{"__ext__bk_46__container_name":"unify-query","_value_":3533,"_timestamp_":1744677990000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744678020000},{"__ext__bk_46__container_name":"unify-query","_value_":3556,"_timestamp_":1744678050000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744678080000},{"__ext__bk_46__container_name":"unify-query","_value_":3664,"_timestamp_":1744678110000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744678140000},{"__ext__bk_46__container_name":"unify-query","_value_":3222,"_timestamp_":1744678170000},{"__ext__bk_46__container_name":"unify-query","_value_":4750,"_timestamp_":1744678200000},{"__ext__bk_46__container_name":"unify-query","_value_":3546,"_timestamp_":1744678230000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744678260000},{"__ext__bk_46__container_name":"unify-query","_value_":3702,"_timestamp_":1744678290000},{"__ext__bk_46__container_name":"unify-query","_value_":4564,"_timestamp_":1744678320000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744678350000},{"__ext__bk_46__container_name":"unify-query","_value_":4130,"_timestamp_":1744678380000},{"__ext__bk_46__container_name":"unify-query","_value_":3412,"_timestamp_":1744678410000},{"__ext__bk_46__container_name":"unify-query","_value_":4614,"_timestamp_":1744678440000},{"__ext__bk_46__container_name":"unify-query","_value_":3522,"_timestamp_":1744678470000},{"__ext__bk_46__container_name":"unify-query","_value_":4148,"_timestamp_":1744678500000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744678530000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744678560000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744678590000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744678620000},{"__ext__bk_46__container_name":"unify-query","_value_":3529,"_timestamp_":1744678650000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744678680000},{"__ext__bk_46__container_name":"unify-query","_value_":3487,"_timestamp_":1744678710000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744678740000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744678770000},{"__ext__bk_46__container_name":"unify-query","_value_":7230,"_timestamp_":1744678800000},{"__ext__bk_46__container_name":"unify-query","_value_":3818,"_timestamp_":1744678830000},{"__ext__bk_46__container_name":"unify-query","_value_":11924,"_timestamp_":1744678860000},{"__ext__bk_46__container_name":"unify-query","_value_":27269,"_timestamp_":1744678890000},{"__ext__bk_46__container_name":"unify-query","_value_":5073,"_timestamp_":1744678920000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744678950000},{"__ext__bk_46__container_name":"unify-query","_value_":4474,"_timestamp_":1744678980000},{"__ext__bk_46__container_name":"unify-query","_value_":3536,"_timestamp_":1744679010000},{"__ext__bk_46__container_name":"unify-query","_value_":4525,"_timestamp_":1744679040000},{"__ext__bk_46__container_name":"unify-query","_value_":3503,"_timestamp_":1744679070000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744679100000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744679130000},{"__ext__bk_46__container_name":"unify-query","_value_":4259,"_timestamp_":1744679160000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744679190000},{"__ext__bk_46__container_name":"unify-query","_value_":4218,"_timestamp_":1744679220000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744679250000},{"__ext__bk_46__container_name":"unify-query","_value_":4417,"_timestamp_":1744679280000},{"__ext__bk_46__container_name":"unify-query","_value_":3730,"_timestamp_":1744679310000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744679340000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744679370000},{"__ext__bk_46__container_name":"unify-query","_value_":4849,"_timestamp_":1744679400000},{"__ext__bk_46__container_name":"unify-query","_value_":3803,"_timestamp_":1744679430000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744679460000},{"__ext__bk_46__container_name":"unify-query","_value_":3674,"_timestamp_":1744679490000},{"__ext__bk_46__container_name":"unify-query","_value_":4727,"_timestamp_":1744679520000},{"__ext__bk_46__container_name":"unify-query","_value_":3926,"_timestamp_":1744679550000},{"__ext__bk_46__container_name":"unify-query","_value_":4173,"_timestamp_":1744679580000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744679610000},{"__ext__bk_46__container_name":"unify-query","_value_":4968,"_timestamp_":1744679640000},{"__ext__bk_46__container_name":"unify-query","_value_":3432,"_timestamp_":1744679670000},{"__ext__bk_46__container_name":"unify-query","_value_":5059,"_timestamp_":1744679700000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744679730000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744679760000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744679790000},{"__ext__bk_46__container_name":"unify-query","_value_":4436,"_timestamp_":1744679820000},{"__ext__bk_46__container_name":"unify-query","_value_":5299,"_timestamp_":1744679850000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744679880000},{"__ext__bk_46__container_name":"unify-query","_value_":3861,"_timestamp_":1744679910000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744679940000},{"__ext__bk_46__container_name":"unify-query","_value_":3711,"_timestamp_":1744679970000},{"__ext__bk_46__container_name":"unify-query","_value_":6021,"_timestamp_":1744680000000},{"__ext__bk_46__container_name":"unify-query","_value_":3942,"_timestamp_":1744680030000},{"__ext__bk_46__container_name":"unify-query","_value_":4800,"_timestamp_":1744680060000},{"__ext__bk_46__container_name":"unify-query","_value_":3681,"_timestamp_":1744680090000},{"__ext__bk_46__container_name":"unify-query","_value_":4592,"_timestamp_":1744680120000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744680150000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744680180000},{"__ext__bk_46__container_name":"unify-query","_value_":3490,"_timestamp_":1744680210000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744680240000},{"__ext__bk_46__container_name":"unify-query","_value_":4009,"_timestamp_":1744680270000},{"__ext__bk_46__container_name":"unify-query","_value_":4837,"_timestamp_":1744680300000},{"__ext__bk_46__container_name":"unify-query","_value_":3227,"_timestamp_":1744680330000},{"__ext__bk_46__container_name":"unify-query","_value_":4531,"_timestamp_":1744680360000},{"__ext__bk_46__container_name":"unify-query","_value_":2888,"_timestamp_":1744680390000},{"__ext__bk_46__container_name":"unify-query","_value_":5083,"_timestamp_":1744680420000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744680450000},{"__ext__bk_46__container_name":"unify-query","_value_":4207,"_timestamp_":1744680480000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744680510000},{"__ext__bk_46__container_name":"unify-query","_value_":4482,"_timestamp_":1744680540000},{"__ext__bk_46__container_name":"unify-query","_value_":3110,"_timestamp_":1744680570000},{"__ext__bk_46__container_name":"unify-query","_value_":13551,"_timestamp_":1744680600000},{"__ext__bk_46__container_name":"unify-query","_value_":17159,"_timestamp_":1744680630000},{"__ext__bk_46__container_name":"unify-query","_value_":6284,"_timestamp_":1744680660000},{"__ext__bk_46__container_name":"unify-query","_value_":9924,"_timestamp_":1744680690000},{"__ext__bk_46__container_name":"unify-query","_value_":4547,"_timestamp_":1744680720000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744680750000},{"__ext__bk_46__container_name":"unify-query","_value_":4312,"_timestamp_":1744680780000},{"__ext__bk_46__container_name":"unify-query","_value_":3689,"_timestamp_":1744680810000},{"__ext__bk_46__container_name":"unify-query","_value_":4680,"_timestamp_":1744680840000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744680870000},{"__ext__bk_46__container_name":"unify-query","_value_":4886,"_timestamp_":1744680900000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744680930000},{"__ext__bk_46__container_name":"unify-query","_value_":4810,"_timestamp_":1744680960000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744680990000},{"__ext__bk_46__container_name":"unify-query","_value_":4594,"_timestamp_":1744681020000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744681050000},{"__ext__bk_46__container_name":"unify-query","_value_":4562,"_timestamp_":1744681080000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744681110000},{"__ext__bk_46__container_name":"unify-query","_value_":5243,"_timestamp_":1744681140000},{"__ext__bk_46__container_name":"unify-query","_value_":5135,"_timestamp_":1744681170000},{"__ext__bk_46__container_name":"unify-query","_value_":6671,"_timestamp_":1744681200000},{"__ext__bk_46__container_name":"unify-query","_value_":3806,"_timestamp_":1744681230000},{"__ext__bk_46__container_name":"unify-query","_value_":4535,"_timestamp_":1744681260000},{"__ext__bk_46__container_name":"unify-query","_value_":3721,"_timestamp_":1744681290000},{"__ext__bk_46__container_name":"unify-query","_value_":4799,"_timestamp_":1744681320000},{"__ext__bk_46__container_name":"unify-query","_value_":3909,"_timestamp_":1744681350000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744681380000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744681410000},{"__ext__bk_46__container_name":"unify-query","_value_":4359,"_timestamp_":1744681440000},{"__ext__bk_46__container_name":"unify-query","_value_":4063,"_timestamp_":1744681470000},{"__ext__bk_46__container_name":"unify-query","_value_":5231,"_timestamp_":1744681500000},{"__ext__bk_46__container_name":"unify-query","_value_":3778,"_timestamp_":1744681530000},{"__ext__bk_46__container_name":"unify-query","_value_":4684,"_timestamp_":1744681560000},{"__ext__bk_46__container_name":"unify-query","_value_":4072,"_timestamp_":1744681590000},{"__ext__bk_46__container_name":"unify-query","_value_":5029,"_timestamp_":1744681620000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744681650000},{"__ext__bk_46__container_name":"unify-query","_value_":4670,"_timestamp_":1744681680000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744681710000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744681740000},{"__ext__bk_46__container_name":"unify-query","_value_":3041,"_timestamp_":1744681770000},{"__ext__bk_46__container_name":"unify-query","_value_":5043,"_timestamp_":1744681800000},{"__ext__bk_46__container_name":"unify-query","_value_":3530,"_timestamp_":1744681830000},{"__ext__bk_46__container_name":"unify-query","_value_":6807,"_timestamp_":1744681860000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744681890000},{"__ext__bk_46__container_name":"unify-query","_value_":6841,"_timestamp_":1744681920000},{"__ext__bk_46__container_name":"unify-query","_value_":4519,"_timestamp_":1744681950000},{"__ext__bk_46__container_name":"unify-query","_value_":6617,"_timestamp_":1744681980000},{"__ext__bk_46__container_name":"unify-query","_value_":4633,"_timestamp_":1744682010000},{"__ext__bk_46__container_name":"unify-query","_value_":5997,"_timestamp_":1744682040000},{"__ext__bk_46__container_name":"unify-query","_value_":4446,"_timestamp_":1744682070000},{"__ext__bk_46__container_name":"unify-query","_value_":5569,"_timestamp_":1744682100000},{"__ext__bk_46__container_name":"unify-query","_value_":4324,"_timestamp_":1744682130000},{"__ext__bk_46__container_name":"unify-query","_value_":5354,"_timestamp_":1744682160000},{"__ext__bk_46__container_name":"unify-query","_value_":7245,"_timestamp_":1744682190000},{"__ext__bk_46__container_name":"unify-query","_value_":5258,"_timestamp_":1744682220000},{"__ext__bk_46__container_name":"unify-query","_value_":4296,"_timestamp_":1744682250000},{"__ext__bk_46__container_name":"unify-query","_value_":5349,"_timestamp_":1744682280000},{"__ext__bk_46__container_name":"unify-query","_value_":4479,"_timestamp_":1744682310000},{"__ext__bk_46__container_name":"unify-query","_value_":5127,"_timestamp_":1744682340000},{"__ext__bk_46__container_name":"unify-query","_value_":4006,"_timestamp_":1744682370000},{"__ext__bk_46__container_name":"unify-query","_value_":19058,"_timestamp_":1744682400000},{"__ext__bk_46__container_name":"unify-query","_value_":14501,"_timestamp_":1744682430000},{"__ext__bk_46__container_name":"unify-query","_value_":3810,"_timestamp_":1744682460000},{"__ext__bk_46__container_name":"unify-query","_value_":12368,"_timestamp_":1744682490000},{"__ext__bk_46__container_name":"unify-query","_value_":6976,"_timestamp_":1744682520000},{"__ext__bk_46__container_name":"unify-query","_value_":4399,"_timestamp_":1744682550000},{"__ext__bk_46__container_name":"unify-query","_value_":5482,"_timestamp_":1744682580000},{"__ext__bk_46__container_name":"unify-query","_value_":4524,"_timestamp_":1744682610000},{"__ext__bk_46__container_name":"unify-query","_value_":5478,"_timestamp_":1744682640000},{"__ext__bk_46__container_name":"unify-query","_value_":4920,"_timestamp_":1744682670000},{"__ext__bk_46__container_name":"unify-query","_value_":5347,"_timestamp_":1744682700000},{"__ext__bk_46__container_name":"unify-query","_value_":4427,"_timestamp_":1744682730000},{"__ext__bk_46__container_name":"unify-query","_value_":5102,"_timestamp_":1744682760000},{"__ext__bk_46__container_name":"unify-query","_value_":4441,"_timestamp_":1744682790000},{"__ext__bk_46__container_name":"unify-query","_value_":5596,"_timestamp_":1744682820000},{"__ext__bk_46__container_name":"unify-query","_value_":4888,"_timestamp_":1744682850000},{"__ext__bk_46__container_name":"unify-query","_value_":5306,"_timestamp_":1744682880000},{"__ext__bk_46__container_name":"unify-query","_value_":4825,"_timestamp_":1744682910000},{"__ext__bk_46__container_name":"unify-query","_value_":5897,"_timestamp_":1744682940000},{"__ext__bk_46__container_name":"unify-query","_value_":4481,"_timestamp_":1744682970000},{"__ext__bk_46__container_name":"unify-query","_value_":6086,"_timestamp_":1744683000000},{"__ext__bk_46__container_name":"unify-query","_value_":4910,"_timestamp_":1744683030000},{"__ext__bk_46__container_name":"unify-query","_value_":5676,"_timestamp_":1744683060000},{"__ext__bk_46__container_name":"unify-query","_value_":3626,"_timestamp_":1744683090000},{"__ext__bk_46__container_name":"unify-query","_value_":6929,"_timestamp_":1744683120000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744683150000},{"__ext__bk_46__container_name":"unify-query","_value_":5525,"_timestamp_":1744683180000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744683210000},{"__ext__bk_46__container_name":"unify-query","_value_":5617,"_timestamp_":1744683240000},{"__ext__bk_46__container_name":"unify-query","_value_":4503,"_timestamp_":1744683270000},{"__ext__bk_46__container_name":"unify-query","_value_":6328,"_timestamp_":1744683300000},{"__ext__bk_46__container_name":"unify-query","_value_":4557,"_timestamp_":1744683330000},{"__ext__bk_46__container_name":"unify-query","_value_":5356,"_timestamp_":1744683360000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744683390000},{"__ext__bk_46__container_name":"unify-query","_value_":5335,"_timestamp_":1744683420000},{"__ext__bk_46__container_name":"unify-query","_value_":4640,"_timestamp_":1744683450000},{"__ext__bk_46__container_name":"unify-query","_value_":5399,"_timestamp_":1744683480000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744683510000},{"__ext__bk_46__container_name":"unify-query","_value_":5415,"_timestamp_":1744683540000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744683570000},{"__ext__bk_46__container_name":"unify-query","_value_":6949,"_timestamp_":1744683600000},{"__ext__bk_46__container_name":"unify-query","_value_":4574,"_timestamp_":1744683630000},{"__ext__bk_46__container_name":"unify-query","_value_":5757,"_timestamp_":1744683660000},{"__ext__bk_46__container_name":"unify-query","_value_":4669,"_timestamp_":1744683690000},{"__ext__bk_46__container_name":"unify-query","_value_":5706,"_timestamp_":1744683720000},{"__ext__bk_46__container_name":"unify-query","_value_":4472,"_timestamp_":1744683750000},{"__ext__bk_46__container_name":"unify-query","_value_":5386,"_timestamp_":1744683780000},{"__ext__bk_46__container_name":"unify-query","_value_":4490,"_timestamp_":1744683810000},{"__ext__bk_46__container_name":"unify-query","_value_":5104,"_timestamp_":1744683840000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744683870000},{"__ext__bk_46__container_name":"unify-query","_value_":5979,"_timestamp_":1744683900000},{"__ext__bk_46__container_name":"unify-query","_value_":4853,"_timestamp_":1744683930000},{"__ext__bk_46__container_name":"unify-query","_value_":6691,"_timestamp_":1744683960000},{"__ext__bk_46__container_name":"unify-query","_value_":4572,"_timestamp_":1744683990000},{"__ext__bk_46__container_name":"unify-query","_value_":5554,"_timestamp_":1744684020000},{"__ext__bk_46__container_name":"unify-query","_value_":5244,"_timestamp_":1744684050000},{"__ext__bk_46__container_name":"unify-query","_value_":5392,"_timestamp_":1744684080000},{"__ext__bk_46__container_name":"unify-query","_value_":4550,"_timestamp_":1744684110000},{"__ext__bk_46__container_name":"unify-query","_value_":520,"_timestamp_":1744684140000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":52,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":8,"connect_db":55,"match_query_routing_rule":0,"check_permission":73,"check_query_semantic":0,"pick_valid_storage":1},"total_record_size":269248,"timetaken":0.191,"result_schema":[{"field_type":"string","field_name":"__c0","field_alias":"__ext__bk_46__container_name","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1},{"field_type":"long","field_name":"__c2","field_alias":"_timestamp_","field_index":2}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + }) + + for i, c := range map[string]struct { + queryTs *structured.QueryTs + result string + }{ + "查询 1 条原始数据，按照字段正向排序": { + queryTs: &structured.QueryTs{ + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(tableID), + FieldName: "gseIndex", + Limit: 1, + From: 0, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "_value", + }, + MetricMerge: "a", + Start: strconv.FormatInt(defaultStart.Unix(), 10), + End: strconv.FormatInt(defaultEnd.Unix(), 10), + Instant: false, + SpaceUid: spaceUid, + }, + result: `{ + "series" : [ { + "name" : "_result0", + "metric_name" : "", + "columns" : [ "_time", "_value" ], + "types" : [ "float", "float" ], + "group_keys" : [ "__ext.container_id", "__ext.container_image", "__ext.container_name", "__ext.io_kubernetes_pod", "__ext.io_kubernetes_pod_ip", "__ext.io_kubernetes_pod_namespace", "__ext.io_kubernetes_pod_uid", "__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type", "__name__", "__shard_key__", "cloudid", "dteventtime", "dteventtimestamp", "file", "gseindex", "iterationindex", "level", "localtime", "log", "message", "path", "report_time", "serverip", "thedate", "time", "trace_id" ], + "group_values" : [ "375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2", "sha256:3a0506f06f1467e93c3a582203aac1a7501e77091572ec9612ddeee4a4dbbdb8", "unify-query", "bk-datalink-unify-query-6df8bcc4c9-rk4sc", "127.0.0.1", "blueking", "558c5b17-b221-47e1-aa66-036cc9b43e2a", "bk-datalink-unify-query-6df8bcc4c9", "ReplicaSet", "bklog:result_table:doris:gseIndex", "29077703953", "0", "", "1744662180000", "http/handler.go:320", "2450131", "19", "info", "", "2025-04-14T20:22:59.982Z\tinfo\thttp/handler.go:320\t[5108397435e997364f8dc1251533e65e] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}", " header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}", "/var/host/data/bcs/lib/docker/containers/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2-json.log", "2025-04-14T20:22:59.982Z", "30.189.37.111", "20250415", "1744662180000", "5108397435e997364f8dc1251533e65e" ], + "values" : [ [ 1744662480000, 2450131 ] ] + } ] +}`, + }, + "根据维度 __ext.container_name 进行 count 聚合，同时用值正向排序": { + queryTs: &structured.QueryTs{ + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(tableID), + FieldName: "gseIndex", + ReferenceName: "a", + TimeAggregation: structured.TimeAggregation{ + Function: "count_over_time", + Window: "30s", + }, + AggregateMethodList: structured.AggregateMethodList{ + { + Method: "sum", + Dimensions: []string{"__ext.container_name"}, + }, + { + Method: "topk", + VArgsList: []interface{}{ + 5, + }, + }, + }, + }, + }, + OrderBy: structured.OrderBy{ + "_value", + }, + MetricMerge: "a", + Start: strconv.FormatInt(defaultStart.Unix(), 10), + End: strconv.FormatInt(defaultEnd.Unix(), 10), + Instant: false, + SpaceUid: spaceUid, + Step: "30s", + }, + result: `{ + "series" : [ { + "name" : "_result0", + "metric_name" : "", + "columns" : [ "_time", "_value" ], + "types" : [ "float", "float" ], + "group_keys" : [ "__ext.container_name" ], + "group_values" : [ "unify-query" ], + "values" : [ [ 1744662510000, 3684 ], [ 1744662540000, 4012 ], [ 1744662570000, 3671 ], [ 1744662600000, 17092 ], [ 1744662630000, 12881 ], [ 1744662660000, 5902 ], [ 1744662690000, 10443 ], [ 1744662720000, 4388 ], [ 1744662750000, 3357 ], [ 1744662780000, 4381 ], [ 1744662810000, 3683 ], [ 1744662840000, 4353 ], [ 1744662870000, 3441 ], [ 1744662900000, 4251 ], [ 1744662930000, 3476 ], [ 1744662960000, 4036 ], [ 1744662990000, 3549 ], [ 1744663020000, 4351 ], [ 1744663050000, 3651 ], [ 1744663080000, 4096 ], [ 1744663110000, 3618 ], [ 1744663140000, 4100 ], [ 1744663170000, 3622 ], [ 1744663200000, 6044 ], [ 1744663230000, 3766 ], [ 1744663260000, 4461 ], [ 1744663290000, 3783 ], [ 1744663320000, 4559 ], [ 1744663350000, 3634 ], [ 1744663380000, 3869 ], [ 1744663410000, 3249 ], [ 1744663440000, 4473 ], [ 1744663470000, 3514 ], [ 1744663500000, 4923 ], [ 1744663530000, 3379 ], [ 1744663560000, 4489 ], [ 1744663590000, 3411 ], [ 1744663620000, 4374 ], [ 1744663650000, 3370 ], [ 1744663680000, 4310 ], [ 1744663710000, 3609 ], [ 1744663740000, 4318 ], [ 1744663770000, 3570 ], [ 1744663800000, 4334 ], [ 1744663830000, 3767 ], [ 1744663860000, 4455 ], [ 1744663890000, 3703 ], [ 1744663920000, 4511 ], [ 1744663950000, 3667 ], [ 1744663980000, 3998 ], [ 1744664010000, 3579 ], [ 1744664040000, 4156 ], [ 1744664070000, 3340 ], [ 1744664100000, 4344 ], [ 1744664130000, 3590 ], [ 1744664160000, 4161 ], [ 1744664190000, 3484 ], [ 1744664220000, 4273 ], [ 1744664250000, 3494 ], [ 1744664280000, 4230 ], [ 1744664310000, 3619 ], [ 1744664340000, 4013 ], [ 1744664370000, 3565 ], [ 1744664400000, 18144 ], [ 1744664430000, 13615 ], [ 1744664460000, 3178 ], [ 1744664490000, 13044 ], [ 1744664520000, 4767 ], [ 1744664550000, 3528 ], [ 1744664580000, 4316 ], [ 1744664610000, 3317 ], [ 1744664640000, 4395 ], [ 1744664670000, 3599 ], [ 1744664700000, 4149 ], [ 1744664730000, 3474 ], [ 1744664760000, 4201 ], [ 1744664790000, 3384 ], [ 1744664820000, 4442 ], [ 1744664850000, 3559 ], [ 1744664880000, 4166 ], [ 1744664910000, 3438 ], [ 1744664940000, 4244 ], [ 1744664970000, 3640 ], [ 1744665000000, 4305 ], [ 1744665030000, 3771 ], [ 1744665060000, 4485 ], [ 1744665090000, 3842 ], [ 1744665120000, 4423 ], [ 1744665150000, 3610 ], [ 1744665180000, 4125 ], [ 1744665210000, 3500 ], [ 1744665240000, 4252 ], [ 1744665270000, 3427 ], [ 1744665300000, 5089 ], [ 1744665330000, 3450 ], [ 1744665360000, 4349 ], [ 1744665390000, 3188 ], [ 1744665420000, 4556 ], [ 1744665450000, 3372 ], [ 1744665480000, 4408 ], [ 1744665510000, 3445 ], [ 1744665540000, 4213 ], [ 1744665570000, 3408 ], [ 1744665600000, 6235 ], [ 1744665630000, 3641 ], [ 1744665660000, 4577 ], [ 1744665690000, 3719 ], [ 1744665720000, 4548 ], [ 1744665750000, 3420 ], [ 1744665780000, 4246 ], [ 1744665810000, 3359 ], [ 1744665840000, 4332 ], [ 1744665870000, 3422 ], [ 1744665900000, 4229 ], [ 1744665930000, 3610 ], [ 1744665960000, 4119 ], [ 1744665990000, 3570 ], [ 1744666020000, 4144 ], [ 1744666050000, 3302 ], [ 1744666080000, 4398 ], [ 1744666110000, 3559 ], [ 1744666140000, 4097 ], [ 1744666170000, 3315 ], [ 1744666200000, 16721 ], [ 1744666230000, 13631 ], [ 1744666260000, 2982 ], [ 1744666290000, 11858 ], [ 1744666320000, 5515 ], [ 1744666350000, 2869 ], [ 1744666380000, 4795 ], [ 1744666410000, 3603 ], [ 1744666440000, 4204 ], [ 1744666470000, 3264 ], [ 1744666500000, 4377 ], [ 1744666530000, 3443 ], [ 1744666560000, 4307 ], [ 1744666590000, 3459 ], [ 1744666620000, 4342 ], [ 1744666650000, 3598 ], [ 1744666680000, 4052 ], [ 1744666710000, 3577 ], [ 1744666740000, 4128 ], [ 1744666770000, 3499 ], [ 1744666800000, 6209 ], [ 1744666830000, 3575 ], [ 1744666860000, 4543 ], [ 1744666890000, 3604 ], [ 1744666920000, 4579 ], [ 1744666950000, 3531 ], [ 1744666980000, 4314 ], [ 1744667010000, 3416 ], [ 1744667040000, 4320 ], [ 1744667070000, 3488 ], [ 1744667100000, 5054 ], [ 1744667130000, 3525 ], [ 1744667160000, 4313 ], [ 1744667190000, 3607 ], [ 1744667220000, 4118 ], [ 1744667250000, 3350 ], [ 1744667280000, 4280 ], [ 1744667310000, 3634 ], [ 1744667340000, 4174 ], [ 1744667370000, 3807 ], [ 1744667400000, 4358 ], [ 1744667430000, 3595 ], [ 1744667460000, 4630 ], [ 1744667490000, 3845 ], [ 1744667520000, 4361 ], [ 1744667550000, 3572 ], [ 1744667580000, 4095 ], [ 1744667610000, 3535 ], [ 1744667640000, 4200 ], [ 1744667670000, 3390 ], [ 1744667700000, 4262 ], [ 1744667730000, 3398 ], [ 1744667760000, 4320 ], [ 1744667790000, 3429 ], [ 1744667820000, 4288 ], [ 1744667850000, 3482 ], [ 1744667880000, 4166 ], [ 1744667910000, 3612 ], [ 1744667940000, 4194 ], [ 1744667970000, 3423 ], [ 1744668000000, 18203 ], [ 1744668030000, 13685 ], [ 1744668060000, 3281 ], [ 1744668090000, 12556 ], [ 1744668120000, 4893 ], [ 1744668150000, 3607 ], [ 1744668180000, 4336 ], [ 1744668210000, 3609 ], [ 1744668240000, 4097 ], [ 1744668270000, 3669 ], [ 1744668300000, 3997 ], [ 1744668330000, 3494 ], [ 1744668360000, 4172 ], [ 1744668390000, 3523 ], [ 1744668420000, 3877 ], [ 1744668450000, 3565 ], [ 1744668480000, 4230 ], [ 1744668510000, 3469 ], [ 1744668540000, 4243 ], [ 1744668570000, 3304 ], [ 1744668600000, 4690 ], [ 1744668630000, 3717 ], [ 1744668660000, 4618 ], [ 1744668690000, 3732 ], [ 1744668720000, 4477 ], [ 1744668750000, 3615 ], [ 1744668780000, 4154 ], [ 1744668810000, 3367 ], [ 1744668840000, 4193 ], [ 1744668870000, 3592 ], [ 1744668900000, 4971 ], [ 1744668930000, 3359 ], [ 1744668960000, 4540 ], [ 1744668990000, 3406 ], [ 1744669020000, 4375 ], [ 1744669050000, 3386 ], [ 1744669080000, 4281 ], [ 1744669110000, 3410 ], [ 1744669140000, 4545 ], [ 1744669170000, 3724 ], [ 1744669200000, 5903 ], [ 1744669230000, 3672 ], [ 1744669260000, 4413 ], [ 1744669290000, 3792 ], [ 1744669320000, 4422 ], [ 1744669350000, 3718 ], [ 1744669380000, 4213 ], [ 1744669410000, 3622 ], [ 1744669440000, 4043 ], [ 1744669470000, 3542 ], [ 1744669500000, 4179 ], [ 1744669530000, 3368 ], [ 1744669560000, 4354 ], [ 1744669590000, 3368 ], [ 1744669620000, 4229 ], [ 1744669650000, 3458 ], [ 1744669680000, 4310 ], [ 1744669710000, 3512 ], [ 1744669740000, 4188 ], [ 1744669770000, 3436 ], [ 1744669800000, 12171 ], [ 1744669830000, 18129 ], [ 1744669860000, 7142 ], [ 1744669890000, 9153 ], [ 1744669920000, 4566 ], [ 1744669950000, 3225 ], [ 1744669980000, 4378 ], [ 1744670010000, 3623 ], [ 1744670040000, 4266 ], [ 1744670070000, 3645 ], [ 1744670100000, 4043 ], [ 1744670130000, 3350 ], [ 1744670160000, 4333 ], [ 1744670190000, 3489 ], [ 1744670220000, 4303 ], [ 1744670250000, 3560 ], [ 1744670280000, 4121 ], [ 1744670310000, 3374 ], [ 1744670340000, 4362 ], [ 1744670370000, 3242 ], [ 1744670400000, 6416 ], [ 1744670430000, 3697 ], [ 1744670460000, 4506 ], [ 1744670490000, 3749 ], [ 1744670520000, 4587 ], [ 1744670550000, 3538 ], [ 1744670580000, 4221 ], [ 1744670610000, 3476 ], [ 1744670640000, 4227 ], [ 1744670670000, 3587 ], [ 1744670700000, 4848 ], [ 1744670730000, 3551 ], [ 1744670760000, 4068 ], [ 1744670790000, 3387 ], [ 1744670820000, 4366 ], [ 1744670850000, 3635 ], [ 1744670880000, 4256 ], [ 1744670910000, 3690 ], [ 1744670940000, 4155 ], [ 1744670970000, 3318 ], [ 1744671000000, 4661 ], [ 1744671030000, 3494 ], [ 1744671060000, 4442 ], [ 1744671090000, 3643 ], [ 1744671120000, 4755 ], [ 1744671150000, 3607 ], [ 1744671180000, 4284 ], [ 1744671210000, 3258 ], [ 1744671240000, 4453 ], [ 1744671270000, 3431 ], [ 1744671300000, 4231 ], [ 1744671330000, 3623 ], [ 1744671360000, 3907 ], [ 1744671390000, 3524 ], [ 1744671420000, 4438 ], [ 1744671450000, 3547 ], [ 1744671480000, 4033 ], [ 1744671510000, 3632 ], [ 1744671540000, 4162 ], [ 1744671570000, 3588 ], [ 1744671600000, 16444 ], [ 1744671630000, 15396 ], [ 1744671660000, 3024 ], [ 1744671690000, 12656 ], [ 1744671720000, 4733 ], [ 1744671750000, 3766 ], [ 1744671780000, 4388 ], [ 1744671810000, 3340 ], [ 1744671840000, 4487 ], [ 1744671870000, 3549 ], [ 1744671900000, 4154 ], [ 1744671930000, 3406 ], [ 1744671960000, 4314 ], [ 1744671990000, 3472 ], [ 1744672020000, 4309 ], [ 1744672050000, 3458 ], [ 1744672080000, 4191 ], [ 1744672110000, 3475 ], [ 1744672140000, 4194 ], [ 1744672170000, 3525 ], [ 1744672200000, 4445 ], [ 1744672230000, 3822 ], [ 1744672260000, 4346 ], [ 1744672290000, 3700 ], [ 1744672320000, 4615 ], [ 1744672350000, 3591 ], [ 1744672380000, 4056 ], [ 1744672410000, 3544 ], [ 1744672440000, 4188 ], [ 1744672470000, 3647 ], [ 1744672500000, 4887 ], [ 1744672530000, 3450 ], [ 1744672560000, 4302 ], [ 1744672590000, 3425 ], [ 1744672620000, 4320 ], [ 1744672650000, 3532 ], [ 1744672680000, 4282 ], [ 1744672710000, 3571 ], [ 1744672740000, 4182 ], [ 1744672770000, 3210 ], [ 1744672800000, 6383 ], [ 1744672830000, 3622 ], [ 1744672860000, 4408 ], [ 1744672890000, 3611 ], [ 1744672920000, 4795 ], [ 1744672950000, 3632 ], [ 1744672980000, 4102 ], [ 1744673010000, 3534 ], [ 1744673040000, 4212 ], [ 1744673070000, 3380 ], [ 1744673100000, 4289 ], [ 1744673130000, 3565 ], [ 1744673160000, 4120 ], [ 1744673190000, 3526 ], [ 1744673220000, 4200 ], [ 1744673250000, 3302 ], [ 1744673280000, 4370 ], [ 1744673310000, 3462 ], [ 1744673340000, 4223 ], [ 1744673370000, 3564 ], [ 1744673400000, 12072 ], [ 1744673430000, 17986 ], [ 1744673460000, 4089 ], [ 1744673490000, 12000 ], [ 1744673520000, 4790 ], [ 1744673550000, 3637 ], [ 1744673580000, 4177 ], [ 1744673610000, 3438 ], [ 1744673640000, 4465 ], [ 1744673670000, 3627 ], [ 1744673700000, 4131 ], [ 1744673730000, 3396 ], [ 1744673760000, 4395 ], [ 1744673790000, 3638 ], [ 1744673820000, 4093 ], [ 1744673850000, 3584 ], [ 1744673880000, 4082 ], [ 1744673910000, 3475 ], [ 1744673940000, 4051 ], [ 1744673970000, 3354 ], [ 1744674000000, 6296 ], [ 1744674030000, 3473 ], [ 1744674060000, 4412 ], [ 1744674090000, 3793 ], [ 1744674120000, 4391 ], [ 1744674150000, 3836 ], [ 1744674180000, 4190 ], [ 1744674210000, 3478 ], [ 1744674240000, 4230 ], [ 1744674270000, 3488 ], [ 1744674300000, 4964 ], [ 1744674330000, 3455 ], [ 1744674360000, 4116 ], [ 1744674390000, 3250 ], [ 1744674420000, 4494 ], [ 1744674450000, 3326 ], [ 1744674480000, 4590 ], [ 1744674510000, 3580 ], [ 1744674540000, 4368 ], [ 1744674570000, 3685 ], [ 1744674600000, 4381 ], [ 1744674630000, 3699 ], [ 1744674660000, 4513 ], [ 1744674690000, 3729 ], [ 1744674720000, 4500 ], [ 1744674750000, 3639 ], [ 1744674780000, 4018 ], [ 1744674810000, 3587 ], [ 1744674840000, 4168 ], [ 1744674870000, 3389 ], [ 1744674900000, 4289 ], [ 1744674930000, 3540 ], [ 1744674960000, 4106 ], [ 1744674990000, 3478 ], [ 1744675020000, 4268 ], [ 1744675050000, 3577 ], [ 1744675080000, 4087 ], [ 1744675110000, 3511 ], [ 1744675140000, 4174 ], [ 1744675170000, 3573 ], [ 1744675200000, 17095 ], [ 1744675230000, 14907 ], [ 1744675260000, 6455 ], [ 1744675290000, 9818 ], [ 1744675320000, 5253 ], [ 1744675350000, 3567 ], [ 1744675380000, 4047 ], [ 1744675410000, 3342 ], [ 1744675440000, 4605 ], [ 1744675470000, 3394 ], [ 1744675500000, 4260 ], [ 1744675530000, 3373 ], [ 1744675560000, 4341 ], [ 1744675590000, 3559 ], [ 1744675620000, 4188 ], [ 1744675650000, 3519 ], [ 1744675680000, 4143 ], [ 1744675710000, 3630 ], [ 1744675740000, 4042 ], [ 1744675770000, 3653 ], [ 1744675800000, 4358 ], [ 1744675830000, 3688 ], [ 1744675860000, 4450 ], [ 1744675890000, 3387 ], [ 1744675920000, 4864 ], [ 1744675950000, 3629 ], [ 1744675980000, 4127 ], [ 1744676010000, 3424 ], [ 1744676040000, 4267 ], [ 1744676070000, 3328 ], [ 1744676100000, 5128 ], [ 1744676130000, 3657 ], [ 1744676160000, 4185 ], [ 1744676190000, 3336 ], [ 1744676220000, 4532 ], [ 1744676250000, 3700 ], [ 1744676280000, 4174 ], [ 1744676310000, 3318 ], [ 1744676340000, 4463 ], [ 1744676370000, 3502 ], [ 1744676400000, 6064 ], [ 1744676430000, 3292 ], [ 1744676460000, 4858 ], [ 1744676490000, 3543 ], [ 1744676520000, 4620 ], [ 1744676550000, 3750 ], [ 1744676580000, 4043 ], [ 1744676610000, 3595 ], [ 1744676640000, 4152 ], [ 1744676670000, 3550 ], [ 1744676700000, 4011 ], [ 1744676730000, 3502 ], [ 1744676760000, 4050 ], [ 1744676790000, 3118 ], [ 1744676820000, 4628 ], [ 1744676850000, 3441 ], [ 1744676880000, 4366 ], [ 1744676910000, 3500 ], [ 1744676940000, 4160 ], [ 1744676970000, 3662 ], [ 1744677000000, 11392 ], [ 1744677030000, 18649 ], [ 1744677060000, 7107 ], [ 1744677090000, 9213 ], [ 1744677120000, 4235 ], [ 1744677150000, 3623 ], [ 1744677180000, 4412 ], [ 1744677210000, 3436 ], [ 1744677240000, 4233 ], [ 1744677270000, 3440 ], [ 1744677300000, 4383 ], [ 1744677330000, 3507 ], [ 1744677360000, 4288 ], [ 1744677390000, 3197 ], [ 1744677420000, 4605 ], [ 1744677450000, 3249 ], [ 1744677480000, 4421 ], [ 1744677510000, 2998 ], [ 1744677540000, 4700 ], [ 1744677570000, 3598 ], [ 1744677600000, 5781 ], [ 1744677630000, 3734 ], [ 1744677660000, 4510 ], [ 1744677690000, 3752 ], [ 1744677720000, 4447 ], [ 1744677750000, 3523 ], [ 1744677780000, 4187 ], [ 1744677810000, 3640 ], [ 1744677840000, 3900 ], [ 1744677870000, 3514 ], [ 1744677900000, 4863 ], [ 1744677930000, 3565 ], [ 1744677960000, 4335 ], [ 1744677990000, 3533 ], [ 1744678020000, 4307 ], [ 1744678050000, 3556 ], [ 1744678080000, 4179 ], [ 1744678110000, 3664 ], [ 1744678140000, 4362 ], [ 1744678170000, 3222 ], [ 1744678200000, 4750 ], [ 1744678230000, 3546 ], [ 1744678260000, 4601 ], [ 1744678290000, 3702 ], [ 1744678320000, 4564 ], [ 1744678350000, 3610 ], [ 1744678380000, 4130 ], [ 1744678410000, 3412 ], [ 1744678440000, 4614 ], [ 1744678470000, 3522 ], [ 1744678500000, 4148 ], [ 1744678530000, 3408 ], [ 1744678560000, 4261 ], [ 1744678590000, 3607 ], [ 1744678620000, 4172 ], [ 1744678650000, 3529 ], [ 1744678680000, 4227 ], [ 1744678710000, 3487 ], [ 1744678740000, 4298 ], [ 1744678770000, 3609 ], [ 1744678800000, 7230 ], [ 1744678830000, 3818 ], [ 1744678860000, 11924 ], [ 1744678890000, 27269 ], [ 1744678920000, 5073 ], [ 1744678950000, 3474 ], [ 1744678980000, 4474 ], [ 1744679010000, 3536 ], [ 1744679040000, 4525 ], [ 1744679070000, 3503 ], [ 1744679100000, 4194 ], [ 1744679130000, 3557 ], [ 1744679160000, 4259 ], [ 1744679190000, 3611 ], [ 1744679220000, 4218 ], [ 1744679250000, 3622 ], [ 1744679280000, 4417 ], [ 1744679310000, 3730 ], [ 1744679340000, 4204 ], [ 1744679370000, 3641 ], [ 1744679400000, 4849 ], [ 1744679430000, 3803 ], [ 1744679460000, 4398 ], [ 1744679490000, 3674 ], [ 1744679520000, 4727 ], [ 1744679550000, 3926 ], [ 1744679580000, 4173 ], [ 1744679610000, 3531 ], [ 1744679640000, 4968 ], [ 1744679670000, 3432 ], [ 1744679700000, 5059 ], [ 1744679730000, 3560 ], [ 1744679760000, 4087 ], [ 1744679790000, 3590 ], [ 1744679820000, 4436 ], [ 1744679850000, 5299 ], [ 1744679880000, 4320 ], [ 1744679910000, 3861 ], [ 1744679940000, 4511 ], [ 1744679970000, 3711 ], [ 1744680000000, 6021 ], [ 1744680030000, 3942 ], [ 1744680060000, 4800 ], [ 1744680090000, 3681 ], [ 1744680120000, 4592 ], [ 1744680150000, 3560 ], [ 1744680180000, 4194 ], [ 1744680210000, 3490 ], [ 1744680240000, 4971 ], [ 1744680270000, 4009 ], [ 1744680300000, 4837 ], [ 1744680330000, 3227 ], [ 1744680360000, 4531 ], [ 1744680390000, 2888 ], [ 1744680420000, 5083 ], [ 1744680450000, 3557 ], [ 1744680480000, 4207 ], [ 1744680510000, 3373 ], [ 1744680540000, 4482 ], [ 1744680570000, 3110 ], [ 1744680600000, 13551 ], [ 1744680630000, 17159 ], [ 1744680660000, 6284 ], [ 1744680690000, 9924 ], [ 1744680720000, 4547 ], [ 1744680750000, 3474 ], [ 1744680780000, 4312 ], [ 1744680810000, 3689 ], [ 1744680840000, 4680 ], [ 1744680870000, 3609 ], [ 1744680900000, 4886 ], [ 1744680930000, 3842 ], [ 1744680960000, 4810 ], [ 1744680990000, 4102 ], [ 1744681020000, 4594 ], [ 1744681050000, 4168 ], [ 1744681080000, 4562 ], [ 1744681110000, 4506 ], [ 1744681140000, 5243 ], [ 1744681170000, 5135 ], [ 1744681200000, 6671 ], [ 1744681230000, 3806 ], [ 1744681260000, 4535 ], [ 1744681290000, 3721 ], [ 1744681320000, 4799 ], [ 1744681350000, 3909 ], [ 1744681380000, 4261 ], [ 1744681410000, 3671 ], [ 1744681440000, 4359 ], [ 1744681470000, 4063 ], [ 1744681500000, 5231 ], [ 1744681530000, 3778 ], [ 1744681560000, 4684 ], [ 1744681590000, 4072 ], [ 1744681620000, 5029 ], [ 1744681650000, 3700 ], [ 1744681680000, 4670 ], [ 1744681710000, 3557 ], [ 1744681740000, 4590 ], [ 1744681770000, 3041 ], [ 1744681800000, 5043 ], [ 1744681830000, 3530 ], [ 1744681860000, 6807 ], [ 1744681890000, 4455 ], [ 1744681920000, 6841 ], [ 1744681950000, 4519 ], [ 1744681980000, 6617 ], [ 1744682010000, 4633 ], [ 1744682040000, 5997 ], [ 1744682070000, 4446 ], [ 1744682100000, 5569 ], [ 1744682130000, 4324 ], [ 1744682160000, 5354 ], [ 1744682190000, 7245 ], [ 1744682220000, 5258 ], [ 1744682250000, 4296 ], [ 1744682280000, 5349 ], [ 1744682310000, 4479 ], [ 1744682340000, 5127 ], [ 1744682370000, 4006 ], [ 1744682400000, 19058 ], [ 1744682430000, 14501 ], [ 1744682460000, 3810 ], [ 1744682490000, 12368 ], [ 1744682520000, 6976 ], [ 1744682550000, 4399 ], [ 1744682580000, 5482 ], [ 1744682610000, 4524 ], [ 1744682640000, 5478 ], [ 1744682670000, 4920 ], [ 1744682700000, 5347 ], [ 1744682730000, 4427 ], [ 1744682760000, 5102 ], [ 1744682790000, 4441 ], [ 1744682820000, 5596 ], [ 1744682850000, 4888 ], [ 1744682880000, 5306 ], [ 1744682910000, 4825 ], [ 1744682940000, 5897 ], [ 1744682970000, 4481 ], [ 1744683000000, 6086 ], [ 1744683030000, 4910 ], [ 1744683060000, 5676 ], [ 1744683090000, 3626 ], [ 1744683120000, 6929 ], [ 1744683150000, 4601 ], [ 1744683180000, 5525 ], [ 1744683210000, 4500 ], [ 1744683240000, 5617 ], [ 1744683270000, 4503 ], [ 1744683300000, 6328 ], [ 1744683330000, 4557 ], [ 1744683360000, 5356 ], [ 1744683390000, 4413 ], [ 1744683420000, 5335 ], [ 1744683450000, 4640 ], [ 1744683480000, 5399 ], [ 1744683510000, 4298 ], [ 1744683540000, 5415 ], [ 1744683570000, 4540 ], [ 1744683600000, 6949 ], [ 1744683630000, 4574 ], [ 1744683660000, 5757 ], [ 1744683690000, 4669 ], [ 1744683720000, 5706 ], [ 1744683750000, 4472 ], [ 1744683780000, 5386 ], [ 1744683810000, 4490 ], [ 1744683840000, 5104 ], [ 1744683870000, 4201 ], [ 1744683900000, 5979 ], [ 1744683930000, 4853 ], [ 1744683960000, 6691 ], [ 1744683990000, 4572 ], [ 1744684020000, 5554 ], [ 1744684050000, 5244 ], [ 1744684080000, 5392 ], [ 1744684110000, 4550 ] ] + } ] +}`, + }, + } { + t.Run(fmt.Sprintf("%s", i), func(t *testing.T) { + metadata.SetUser(ctx, "username:test", spaceUid, "true") + + res, err := queryTsWithPromEngine(ctx, c.queryTs) + assert.Nil(t, err) + excepted, err := json.Marshal(res) + assert.Nil(t, err) + assert.JSONEq(t, c.result, string(excepted)) + }) + } +} + func TestQueryTsWithEs(t *testing.T) { ctx := metadata.InitHashID(context.Background()) + viper.Set(bkapi.BkAPIAddressConfigPath, mock.EsUrlDomain) + spaceUid := influxdb.SpaceUid tableID := influxdb.ResultTableEs @@ -133,7 +260,7 @@ func TestQueryTsWithEs(t *testing.T) { } } -func TestQueryReference(t *testing.T) { +func TestQueryReferenceWithEs(t *testing.T) { ctx := metadata.InitHashID(context.Background()) spaceUid := influxdb.SpaceUid @@ -143,7 +270,6 @@ func TestQueryReference(t *testing.T) { promql.MockEngine() influxdb.MockSpaceRouter(ctx) - mock.Init() defaultStart := time.UnixMilli(1741154079123) // 2025-03-05 13:54:39 defaultEnd := time.UnixMilli(1741155879987) // 2025-03-05 14:24:39 @@ -716,6 +842,10 @@ func TestQueryRawWithInstance(t *testing.T) { start := "1723594000" end := "1723595000" + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + }) + mock.Es.Set(map[string]any{ `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, `{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, @@ -756,7 +886,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: "1723595000234", }, total: 1e4, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"e058129ae18bff87c95e83f24584e654","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c124dae69af9b86a7128ee4281820158","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"43a36f412886bf30b0746562513638d3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"0826407be7f04f19086774ed68eac8dd","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"d56b4120194eb37f53410780da777d43","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"e058129ae18bff87c95e83f24584e654","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"c124dae69af9b86a7128ee4281820158","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"43a36f412886bf30b0746562513638d3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"0826407be7f04f19086774ed68eac8dd","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"},{"__data_label":"bkbase_es","__doc_id":"d56b4120194eb37f53410780da777d43","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"}]`, }, "query es with multi rt and multi from 0 - 5": { queryTs: &structured.QueryTs{ @@ -785,7 +915,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { From: function.IntPoint(0), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -794,8 +924,8 @@ func TestQueryRawWithInstance(t *testing.T) { }, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5}}`, + expected: `[{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"from":5},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5}}`, }, "query es with multi rt and multi from 5 - 10": { queryTs: &structured.QueryTs{ @@ -825,7 +955,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { From: function.IntPoint(5), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -834,8 +964,8 @@ func TestQueryRawWithInstance(t *testing.T) { }, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":7},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":8}}`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"from":7},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":8}}`, }, "query es with multi rt and one from 0 - 5": { queryTs: &structured.QueryTs{ @@ -866,8 +996,8 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]}}`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"search_after":["1","5"]},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]}}`, }, "query es with multi rt and one from 5 - 10": { queryTs: &structured.QueryTs{ @@ -898,8 +1028,8 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, - options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]},"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]}}`, + expected: `[{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]},"result_table.es|http://127.0.0.1:93002":{"search_after":["2","5"]}}`, }, "query_bk_base_es_1 to 1": { queryTs: &structured.QueryTs{ @@ -917,7 +1047,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 1e4, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, + expected: `[{"__data_label":"es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, }, "query with scroll - 1": { queryTs: &structured.QueryTs{ @@ -948,9 +1078,9 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, Scroll: "5m", }, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"},{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, total: 246, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"}}`, + options: `{"result_table.es|http://127.0.0.1:93002":{"scroll_id":"one"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"}}`, }, "query with scroll - 2": { queryTs: &structured.QueryTs{ @@ -980,7 +1110,7 @@ func TestQueryRawWithInstance(t *testing.T) { Start: start, End: end, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { ScrollID: "one", }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -989,9 +1119,9 @@ func TestQueryRawWithInstance(t *testing.T) { }, Scroll: "5m", }, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"5"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"},{"__data_label":"es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"3"},{"__data_label":"bkbase_es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"4"},{"__data_label":"es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"4"},{"__data_label":"bkbase_es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"5"},{"__data_label":"es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"5"}]`, total: 246, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"}}`, + options: `{"result_table.es|http://127.0.0.1:93002":{"scroll_id":"two"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"}}`, }, } diff --git a/pkg/unify-query/service/influxdb/hook.go b/pkg/unify-query/service/influxdb/hook.go index fa4521b7a..ec681fad3 100644 --- a/pkg/unify-query/service/influxdb/hook.go +++ b/pkg/unify-query/service/influxdb/hook.go @@ -45,6 +45,8 @@ func setDefaultConfig() { viper.SetDefault(GrpcMaxCallRecvMsgSizeConfigPath, 1024*1024*10) viper.SetDefault(GrpcMaxCallSendMsgSizeConfigPath, 1024*1024*10) + + viper.SetDefault(IsCacheConfigPath, true) } // LoadConfig @@ -71,6 +73,7 @@ func LoadConfig() { SpaceRouterBboltPath = viper.GetString(SpaceRouterBboltPathConfigPath) SpaceRouterBboltBucketName = viper.GetString(SpaceRouterBboltBucketNameConfigPath) SpaceRouterBboltWriteBatchSize = viper.GetInt(SpaceRouterBboltWriteBatchSizeConfigPath) + IsCache = viper.GetBool(IsCacheConfigPath) GrpcMaxCallRecvMsgSize = viper.GetInt(GrpcMaxCallRecvMsgSizeConfigPath) GrpcMaxCallSendMsgSize = viper.GetInt(GrpcMaxCallSendMsgSizeConfigPath) diff --git a/pkg/unify-query/service/influxdb/service.go b/pkg/unify-query/service/influxdb/service.go index f8b41cf12..39bffbf4d 100644 --- a/pkg/unify-query/service/influxdb/service.go +++ b/pkg/unify-query/service/influxdb/service.go @@ -69,19 +69,16 @@ func (s *Service) Reload(ctx context.Context) { err = s.loopReloadStorage(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload es storage failed for->[%s]", err) - return } err = s.loopReloadTableInfo(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload table info failed,error:%s", err) - return } err = s.loopReloadRouter(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload query router failed,error:%s", err) - return } err = s.loopReloadBCSInfo(s.ctx) @@ -313,7 +310,7 @@ func (s *Service) reloadInfluxDBRouter(ctx context.Context) error { // reloadInfluxDBRouter 重新加载 SpaceTsDbRouter func (s *Service) reloadSpaceTsDbRouter(ctx context.Context) error { - ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, true) + ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, IsCache) if err != nil { return err } diff --git a/pkg/unify-query/service/influxdb/settings.go b/pkg/unify-query/service/influxdb/settings.go index 95fe57921..430c7dd15 100644 --- a/pkg/unify-query/service/influxdb/settings.go +++ b/pkg/unify-query/service/influxdb/settings.go @@ -37,6 +37,8 @@ const ( GrpcMaxCallRecvMsgSizeConfigPath = "influxdb.grpc_max_call_recv_msg_size" GrpcMaxCallSendMsgSizeConfigPath = "influxdb.grpc_max_call_send_msg_size" + + IsCacheConfigPath = "influxdb.is_cache" ) var ( @@ -61,6 +63,8 @@ var ( SpaceRouterBboltBucketName string SpaceRouterBboltWriteBatchSize int + IsCache bool + GrpcMaxCallRecvMsgSize int GrpcMaxCallSendMsgSize int ) diff --git a/pkg/unify-query/tsdb/bksql/client_test.go b/pkg/unify-query/tsdb/bksql/client_test.go index 8761c7358..6adedb735 100644 --- a/pkg/unify-query/tsdb/bksql/client_test.go +++ b/pkg/unify-query/tsdb/bksql/client_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -21,15 +21,16 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) var ( - client *Client + client *bksql.Client ) -func MockClient() *Client { +func MockClient() *bksql.Client { if client == nil { - client = (&Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) + client = (&bksql.Client{}).WithUrl(mock.BkBaseUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) } return client @@ -43,10 +44,10 @@ func TestClient_QuerySync(t *testing.T) { end := time.UnixMilli(1729838923416) mock.BkSQL.Set(map[string]any{ - `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &Result{ + `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &bksql.Result{ Result: true, - Code: StatusOK, - Data: &QuerySyncResultData{ + Code: bksql.StatusOK, + Data: &bksql.QuerySyncResultData{ TotalRecords: 5, SelectFieldsOrder: []string{ "dtEventTimeStamp", @@ -94,8 +95,8 @@ func TestClient_QuerySync(t *testing.T) { nil, ) - assert.Equal(t, StatusOK, res.Code) - d, ok := res.Data.(*QuerySyncResultData) + assert.Equal(t, bksql.StatusOK, res.Code) + d, ok := res.Data.(*bksql.QuerySyncResultData) assert.True(t, ok) assert.Equal(t, d.TotalRecords, 5) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index e1c13b0b0..45fbbbf66 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -20,34 +20,35 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) const ( + selectAll = "*" + dtEventTimeStamp = "dtEventTimeStamp" dtEventTime = "dtEventTime" localTime = "localTime" startTime = "_startTime_" endTime = "_endTime_" theDate = "thedate" - - timeStamp = "_timestamp_" - value = "_value_" - - FieldValue = "_value" - FieldTime = "_time" ) var ( internalDimension = map[string]struct{}{ - value: {}, - timeStamp: {}, dtEventTimeStamp: {}, dtEventTime: {}, localTime: {}, startTime: {}, endTime: {}, theDate: {}, + + sqlExpr.TimeStamp: {}, + sqlExpr.Value: {}, } ) @@ -59,24 +60,25 @@ type QueryFactory struct { start time.Time end time.Time - step time.Duration - selects []string - groups []string - orders metadata.Orders + timeAggregate sqlExpr.TimeAggregate - sql strings.Builder + orders metadata.Orders timeField string + + expr sqlExpr.SQLExpr + + highlight *metadata.HighLight } func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f := &QueryFactory{ - ctx: ctx, - query: query, - selects: make([]string, 0), - groups: make([]string, 0), + ctx: ctx, + query: query, + highlight: query.HighLight, } + if query.Orders != nil { f.orders = query.Orders } @@ -86,11 +88,16 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { } else { f.timeField = dtEventTimeStamp } - return f -} -func (f *QueryFactory) write(s string) { - f.sql.WriteString(s + " ") + f.expr = sqlExpr.NewSQLExpr(f.query.Measurement). + WithInternalFields(f.timeField, query.Field). + WithEncode(metadata.GetPromDataFormat(ctx).EncodeFunc()) + + if f.highlight != nil && f.highlight.Enable { + f.expr.IsSetLabels(true) + } + + return f } func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { @@ -99,185 +106,120 @@ func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { return f } -func (f *QueryFactory) ParserQuery() (err error) { - f.lock.Lock() - defer f.lock.Unlock() - - if len(f.query.Aggregates) > 0 { - for _, agg := range f.query.Aggregates { - for _, dim := range agg.Dimensions { - dim = fmt.Sprintf("`%s`", dim) - f.groups = append(f.groups, dim) - f.selects = append(f.selects, dim) - } - f.selects = append(f.selects, fmt.Sprintf("%s(`%s`) AS `%s`", strings.ToUpper(agg.Name), f.query.Field, value)) - if agg.Window > 0 { - // 时间聚合函数兼容时区 - loc, locErr := time.LoadLocation(agg.TimeZone) - if locErr != nil { - loc = time.UTC - } - // 获取时区偏移量 - _, offset := time.Now().In(loc).Zone() - var offsetMillis int - // 只有按天聚合才需要偏移 - if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { - offsetMillis = offset * 1000 - } - - timeField := fmt.Sprintf("(`%s` - ((`%s` - %d) %% %d - %d))", f.timeField, f.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) - - f.groups = append(f.groups, timeField) - f.selects = append(f.selects, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) - f.orders = append(f.orders, metadata.Order{ - Name: FieldTime, - Ast: true, - }) - } - } - } - - if len(f.selects) == 0 { - f.selects = append(f.selects, "*") - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.query.Field, value)) - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.timeField, timeStamp)) - } +func (f *QueryFactory) WithFieldsMap(m map[string]string) *QueryFactory { + f.expr.WithFieldsMap(m) + return f +} - return +func (f *QueryFactory) WithKeepColumns(cols []string) *QueryFactory { + f.expr.WithKeepColumns(cols) + return f } -func (f *QueryFactory) getTheDateFilters() (theDateFilter string, err error) { - // bkbase 使用 时区东八区 转换为 thedate - loc, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - return +func (f *QueryFactory) Table() string { + table := fmt.Sprintf("`%s`", f.query.DB) + if f.query.Measurement != "" { + table += "." + f.query.Measurement } + return table +} - start := f.start.In(loc) - end := f.end.In(loc) - - dates := function.RangeDateWithUnit("day", start, end, 1) - - if len(dates) == 0 { - return - } +func (f *QueryFactory) DescribeTableSQL() string { + return f.expr.DescribeTableSQL(f.Table()) +} - if len(dates) == 1 { - theDateFilter = fmt.Sprintf("`%s` = '%s'", theDate, dates[0]) - return - } +func (f *QueryFactory) FieldMap() map[string]string { + return f.expr.FieldMap() +} - theDateFilter = fmt.Sprintf("`%s` >= '%s' AND `%s` <= '%s'", theDate, dates[0], theDate, dates[len(dates)-1]) - return +func (f *QueryFactory) GetLabelMap() map[string][]string { + return f.expr.GetLabelMap() } -func (f *QueryFactory) SQL() (sql string, err error) { - f.sql.Reset() - err = f.ParserQuery() - if err != nil { +func (f *QueryFactory) HighLight(data map[string]any) (newData map[string]any) { + if f.query.HighLight == nil || !f.query.HighLight.Enable { return } - f.lock.RLock() - defer f.lock.RUnlock() + newData = make(map[string]any) + for k, vs := range f.GetLabelMap() { + if vs == nil { + return + } - f.write("SELECT") - f.write(strings.Join(f.selects, ", ")) - f.write("FROM") - db := fmt.Sprintf("`%s`", f.query.DB) - if f.query.Measurement != "" { - db += "." + f.query.Measurement - } - f.write(db) - f.write("WHERE") - f.write(fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) + if d, ok := data[k]; ok { + var ( + mark1 string + mark2 string + ) + + switch s := d.(type) { + case string: + if f.query.HighLight.MaxAnalyzedOffset > 0 && len(s) > f.query.HighLight.MaxAnalyzedOffset { + mark1 = s[0:f.query.HighLight.MaxAnalyzedOffset] + mark2 = s[f.query.HighLight.MaxAnalyzedOffset:] + } else { + mark1 = s + } - theDateFilter, err := f.getTheDateFilters() - if err != nil { - return - } - if theDateFilter != "" { - f.write("AND") - f.write(theDateFilter) - } + for _, v := range vs { + mark1 = strings.ReplaceAll(mark1, v, fmt.Sprintf("%s", v)) + } - if f.query.BkSqlCondition != "" { - f.write("AND") - f.write("(" + f.query.BkSqlCondition + ")") - } - if len(f.groups) > 0 { - f.write("GROUP BY") - f.write(strings.Join(f.groups, ", ")) - } + res := fmt.Sprintf("%s%s", mark1, mark2) + if res != d { + newData[k] = []string{res} + } + } - orders := make([]string, 0) - for _, order := range f.orders { - var orderField string - switch order.Name { - case FieldValue: - orderField = f.query.Field - case FieldTime: - orderField = timeStamp - default: - orderField = order.Name } - ascName := "ASC" - if !order.Ast { - ascName = "DESC" - } - orders = append(orders, fmt.Sprintf("`%s` %s", orderField, ascName)) - } - if len(orders) > 0 { - sort.Strings(orders) - f.write("ORDER BY") - f.write(strings.Join(orders, ", ")) - } - if f.query.From > 0 { - f.write("OFFSET") - f.write(fmt.Sprintf("%d", f.query.From)) - } - if f.query.Size > 0 { - f.write("LIMIT") - f.write(fmt.Sprintf("%d", f.query.Size)) } - sql = strings.Trim(f.sql.String(), " ") return } -func (f *QueryFactory) dims(dims []string, field string) []string { - dimensions := make([]string, 0) - for _, dim := range dims { - // 判断是否是内置维度，内置维度不是用户上报的维度 - if _, ok := internalDimension[dim]; ok { - continue - } - // 如果是字段值也需要跳过 - if dim == field { - continue +func (f *QueryFactory) ReloadListData(data map[string]any) (newData map[string]any) { + newData = make(map[string]any) + + fieldMap := f.FieldMap() + + for k, d := range data { + if v, ok := fieldMap[k]; ok { + if v == TableTypeVariant { + objectData, err := json.ParseObject(k, d.(string)) + if err != nil { + log.Errorf(f.ctx, "json.ParseObject err: %v", err) + continue + } + for nk, nd := range objectData { + newData[nk] = nd + } + continue + } } - dimensions = append(dimensions, dim) + newData[k] = d } - return dimensions + return } -func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) (*prompb.QueryResult, error) { +func (f *QueryFactory) FormatDataToQueryResult(ctx context.Context, list []map[string]interface{}) (*prompb.QueryResult, error) { res := &prompb.QueryResult{} if len(list) == 0 { return res, nil } - // 维度结构体为空则任务异常 - if len(keys) == 0 { - return res, fmt.Errorf("SelectFieldsOrder is empty") - } - // 获取该指标的维度 key - field := f.query.Field - dimensions := f.dims(keys, field) + encodeFunc := metadata.GetPromDataFormat(ctx).EncodeFunc() + // 获取 metricLabel + metricLabel := f.query.MetricLabels(ctx) + + tsMap := map[string]*prompb.TimeSeries{} + tsTimeMap := make(map[string]map[int64]float64) + isAddZero := f.timeAggregate.Window > 0 && f.expr.Type() == sqlExpr.Doris - tsMap := make(map[string]*prompb.TimeSeries, 0) + // 先获取维度的 key 保证顺序一致 + keys := make([]string, 0) for _, d := range list { // 优先获取时间和值 var ( @@ -294,26 +236,56 @@ func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) continue } - // 获取时间戳，单位是毫秒 - if vtLong, ok = d[timeStamp]; !ok { - vtLong = 0 + nd := f.ReloadListData(d) + if len(keys) == 0 { + for k := range nd { + keys = append(keys, k) + } + sort.Strings(keys) + } + + lbl := make([]prompb.Label, 0) + for _, k := range keys { + switch k { + case sqlExpr.TimeStamp: + if _, ok = nd[k]; ok { + vtLong = nd[k] + } + case sqlExpr.Value: + if _, ok = nd[k]; ok { + vvDouble = nd[k] + } + default: + // 获取维度信息 + val, err := getValue(k, nd) + if err != nil { + log.Errorf(ctx, "get dimension (%s) value error in %+v %s", k, d, err.Error()) + continue + } + + if encodeFunc != nil { + k = encodeFunc(k) + } + + lbl = append(lbl, prompb.Label{ + Name: k, + Value: val, + }) + + } } if vtLong == nil { - continue + vtLong = f.start.UnixMilli() } + switch vtLong.(type) { case int64: vt = vtLong.(int64) case float64: vt = int64(vtLong.(float64)) default: - return res, fmt.Errorf("%s type is error %T, %v", f.timeField, vtLong, vtLong) - } - - // 获取值 - if vvDouble, ok = d[field]; !ok { - return res, fmt.Errorf("dimension %s is emtpy", field) + return res, fmt.Errorf("%s type is error %T, %v", dtEventTimeStamp, vtLong, vtLong) } if vvDouble == nil { @@ -325,23 +297,17 @@ func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) case float64: vv = vvDouble.(float64) default: - return res, fmt.Errorf("%s type is error %T, %v", field, vvDouble, vvDouble) + return res, fmt.Errorf("%s type is error %T, %v", sqlExpr.Value, vvDouble, vvDouble) } - var buf strings.Builder - lbl := make([]prompb.Label, 0, len(dimensions)) - // 获取维度信息 - for _, dimName := range dimensions { - val, err := getValue(dimName, d) - if err != nil { - return res, fmt.Errorf("dimensions %+v %s", dimensions, err.Error()) - } + // 如果是非时间聚合计算，则无需进行指标名的拼接作用 + if metricLabel != nil { + lbl = append(lbl, *metricLabel) + } - buf.WriteString(fmt.Sprintf("%s:%s,", dimName, val)) - lbl = append(lbl, prompb.Label{ - Name: dimName, - Value: val, - }) + var buf strings.Builder + for _, l := range lbl { + buf.WriteString(l.String()) } // 同一个 series 进行合并分组 @@ -353,17 +319,184 @@ func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) } } - tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ - Value: vv, - Timestamp: vt, - }) + // 如果是时间聚合需要进行补零，否则直接返回 + if isAddZero { + if _, ok := tsTimeMap[key]; !ok { + tsTimeMap[key] = make(map[int64]float64) + } + + tsTimeMap[key][vt] = vv + } else { + tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ + Value: vv, + Timestamp: vt, + }) + } } // 转换结构体 res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) - for _, ts := range tsMap { - res.Timeseries = append(res.Timeseries, ts) + + // 如果是时间聚合需要进行补零，否则直接返回 + if isAddZero { + var ( + start time.Time + end time.Time + ) + + startMilli := (f.start.UnixMilli()+f.timeAggregate.OffsetMillis)/f.timeAggregate.Window.Milliseconds()*f.timeAggregate.Window.Milliseconds() - f.timeAggregate.OffsetMillis + start = time.UnixMilli(startMilli) + end = f.end + + for key, ts := range tsMap { + for i := start; end.Sub(i) > 0; i = i.Add(f.timeAggregate.Window) { + sample := prompb.Sample{ + Timestamp: i.UnixMilli(), + Value: 0, + } + if v, ok := tsTimeMap[key][i.UnixMilli()]; ok { + sample.Value = v + } + ts.Samples = append(ts.Samples, sample) + } + res.Timeseries = append(res.Timeseries, ts) + } + } else { + for _, ts := range tsMap { + res.Timeseries = append(res.Timeseries, ts) + } } return res, nil } + +func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { + // bkbase 使用 时区东八区 转换为 thedate + loc, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + return + } + + start := f.start.In(loc) + end := f.end.In(loc) + + dates := function.RangeDateWithUnit("day", start, end, 1) + + if len(dates) == 0 { + return + } + + if len(dates) == 1 { + theDateFilter = fmt.Sprintf("`%s` = '%s'", theDate, dates[0]) + return + } + + theDateFilter = fmt.Sprintf("`%s` >= '%s' AND `%s` <= '%s'", theDate, dates[0], theDate, dates[len(dates)-1]) + return +} + +func (f *QueryFactory) BuildWhere() (string, error) { + var s []string + s = append(s, fmt.Sprintf("`%s` >= %d AND `%s` <= %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) + + theDateFilter, err := f.getTheDateIndexFilters() + if err != nil { + return "", err + } + if theDateFilter != "" { + s = append(s, theDateFilter) + } + + // QueryString to sql + if f.query.QueryString != "" && f.query.QueryString != "*" { + qs, err := f.expr.ParserQueryString(f.query.QueryString) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + // AllConditions to sql + if len(f.query.AllConditions) > 0 { + qs, err := f.expr.ParserAllConditions(f.query.AllConditions) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + return strings.Join(s, " AND "), nil +} + +func (f *QueryFactory) SQL() (sql string, err error) { + var ( + span *trace.Span + ) + + _, span = trace.NewSpan(f.ctx, "make-sql") + defer span.End(&err) + + selectFields, groupFields, orderFields, timeAggregate, err := f.expr.ParserAggregatesAndOrders(f.query.Aggregates, f.orders) + if err != nil { + return + } + + f.timeAggregate = timeAggregate + + span.Set("select-fields", selectFields) + span.Set("group-fields", groupFields) + span.Set("order-fields", orderFields) + span.Set("timeAggregate", timeAggregate) + + sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), f.Table()) + whereString, err := f.BuildWhere() + + span.Set("where-string", whereString) + + if err != nil { + return + } + if whereString != "" { + sql += " WHERE " + whereString + } + if len(groupFields) > 0 { + sql += " GROUP BY " + strings.Join(groupFields, ", ") + } + + if len(orderFields) > 0 { + sort.Strings(orderFields) + sql += " ORDER BY " + strings.Join(orderFields, ", ") + } + if f.query.Size > 0 { + sql += fmt.Sprintf(" LIMIT %d", f.query.Size) + } + if f.query.From > 0 { + sql += fmt.Sprintf(" OFFSET %d", f.query.From) + } + + span.Set("sql", sql) + return +} + +func (f *QueryFactory) dims(dims []string, field string) []string { + dimensions := make([]string, 0) + for _, dim := range dims { + // 判断是否是内置维度，内置维度不是用户上报的维度 + if _, ok := internalDimension[dim]; ok { + continue + } + // 如果是字段值也需要跳过 + if dim == field { + continue + } + + dimensions = append(dimensions, dim) + } + return dimensions +} diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index c94d52c52..144650b98 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -7,10 +7,11 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" + "fmt" "testing" "time" @@ -19,11 +20,12 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) func TestNewSqlFactory(t *testing.T) { - start := time.Unix(1717144141, 0) - end := time.Unix(1717147741, 0) + start := time.Unix(1741795260, 0) + end := time.Unix(1741796260, 0) for name, c := range map[string]struct { query *metadata.Query @@ -32,28 +34,85 @@ func TestNewSqlFactory(t *testing.T) { start time.Time end time.Time }{ - "sum-count_over_time-with-promql-1": { + "doris sum-count_over_time-with-promql-1": { query: &metadata.Query{ - DB: "100133_ieod_logsearch4_errorlog_p", + DB: "5000140_bklog_container_log_demo_analysis", Measurement: "doris", Field: "gseIndex", Aggregates: metadata.Aggregates{ { Name: "count", Dimensions: []string{ - "ip", + "level", }, Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 0, - Orders: metadata.Orders{{Name: "ip", Ast: true}}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + { + DimensionName: "level", + Operator: metadata.ConditionEqual, + Value: []string{"ERROR"}, + }, + }, + }, + From: 0, + Size: 0, + Orders: metadata.Orders{ + { + Name: "level", + Ast: true, + }, + }, + }, + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + }, + "doris sum-count_over_time-with-promql-seconds": { + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{ + "level", + }, + Window: time.Minute + time.Second*15, + }, + }, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + { + DimensionName: "level", + Operator: metadata.ConditionEqual, + Value: []string{"ERROR"}, + }, + }, + }, + From: 0, + Size: 0, + Orders: metadata.Orders{ + { + Name: "level", + Ast: true, + }, + }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC, `ip` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 75000 AS INT) * 75000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, - "sum-with-promql-1": { + "doris sum-with-promql-1": { query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -66,14 +125,22 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 10, - Orders: nil, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + }, + }, + From: 0, + Size: 10, + Orders: nil, }, - expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip` LIMIT 10", + expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, - "count-with-count-promql-1": { + "doris count-with-count-promql-1": { query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -87,11 +154,10 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, - "count-with-count-promql-2": { + "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC // 2024-12-08 05:36:40 Asia/ShangHai start: time.Unix(1733607400, 0), @@ -111,31 +177,73 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "gseIndex", + Value: []string{"0"}, + Operator: metadata.ConditionGt, + }, + }, + }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` <= 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, - "count-with-count-promql-3": { - // 2024-12-07 21:36:40 UTC - // 2024-12-08 05:36:40 Asia/ShangHai - start: time.UnixMilli(1741276799999), - // 2024-12-11 17:49:35 UTC - // 2024-12-12 01:49:35 Asia/ShangHai - end: time.UnixMilli(1741967999999), + "doris count by day with UTC": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), query: &metadata.Query{ - DB: "100680_alpha_server_perf_data_tglog", - Measurement: "", - Field: "sum_Sub8MsFrames", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour * 24, + }, + }, + }, + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + "doris count by day with Asia/Shanghai": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", Aggregates: metadata.Aggregates{ { Name: "count", Window: time.Hour * 24, - TimeZone: "Asia/Shanghai", + TimeZone: "Asia/ShangHai", + }, + }, + }, + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 480) / 1440 AS INT) * 1440 - 480) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + "doris count by dimension with object": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour * 24, + Dimensions: []string{"__ext.container_id"}, }, }, - BkSqlCondition: "`deployment` = 'alpha1-gp-3' and `datacenter` = 'qcloud-tj1'", }, - expected: "SELECT COUNT(`sum_Sub8MsFrames`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 86400000 - 28800000))) AS `_timestamp_` FROM `100680_alpha_server_perf_data_tglog` WHERE `dtEventTimeStamp` >= 1741276799999 AND `dtEventTimeStamp` < 1741967999999 AND `thedate` >= '20250306' AND `thedate` <= '20250314' AND (`deployment` = 'alpha1-gp-3' and `datacenter` = 'qcloud-tj1') GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 86400000 - 28800000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT CAST(__ext[\"container_id\"] AS STRING) AS `__ext__bk_46__container_id`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY __ext__bk_46__container_id, _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { @@ -148,7 +256,7 @@ func TestNewSqlFactory(t *testing.T) { } log.Infof(ctx, "start: %s, end: %s", c.start, c.end) - fact := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) sql, err := fact.SQL() assert.Nil(t, err) assert.Equal(t, c.expected, sql) @@ -182,6 +290,22 @@ func TestWindowWithTimezone(t *testing.T) { expected: time.UnixMilli(1742256000000), }, + { + name: "test window 1d +6 - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/Urumqi", + window: time.Hour * 24, + + expected: time.UnixMilli(1742234400000), + }, + { + name: "test window 1d -6 - 1", + start: time.UnixMilli(1742267704000), + timezone: "America/Guatemala", + window: time.Hour * 24, + + expected: time.UnixMilli(1742191200000), + }, { name: "test window 1d +8 - 1", start: time.UnixMilli(1742267704000), @@ -190,6 +314,14 @@ func TestWindowWithTimezone(t *testing.T) { expected: time.UnixMilli(1742227200000), }, + { + name: "test window 1d +8 - 2", + start: time.UnixMilli(1741885200000), + timezone: "Asia/ShangHai", + window: time.Hour * 24, + + expected: time.UnixMilli(1741881600000), + }, { name: "test window 26h +8 - 1", start: time.UnixMilli(1742267704000), @@ -204,7 +336,7 @@ func TestWindowWithTimezone(t *testing.T) { timezone: "Asia/ShangHai", window: time.Hour * 24 * 3, - expected: time.UnixMilli(1742140800000), + expected: time.UnixMilli(1742054400000), }, { name: "test window 1m +8 - 2", @@ -216,11 +348,11 @@ func TestWindowWithTimezone(t *testing.T) { }, { name: "test window 6h +8 - 2", - start: time.UnixMilli(1742266099000), + start: time.UnixMilli(1742266099000), // 2025-03-18 10:48:19 +0800 timezone: "Asia/ShangHai", window: time.Hour * 6, - expected: time.UnixMilli(1742256000000), + expected: time.UnixMilli(1742256000000), // 2025-03-18 08:00:00 +0800 CST }, } @@ -236,12 +368,54 @@ func TestWindowWithTimezone(t *testing.T) { offset = int64(z) * 1000 } - nTime := cTime - ((cTime-offset)%window - offset) + nTime1 := (cTime + offset) - (cTime+offset)%window - offset + nTime2 := (cTime+offset)/window*window - offset + + var milliUnixToString = func(milliUnix int64) string { + return time.UnixMilli(milliUnix).In(loc).String() + } - actual := time.UnixMilli(nTime) - assert.Equal(t, tc.expected.UnixMilli(), actual.UnixMilli()) - log.Infof(context.TODO(), "%d - ((%d - %d) %% %d - %d", nTime, cTime, offset, window, offset) - log.Infof(context.TODO(), "%s => %s", tc.start.In(loc).String(), actual.In(loc).String()) + log.Infof(context.TODO(), "window: %d, nTime: %s, nTime1: %s nTime2: %s", window/1000, milliUnixToString(cTime), milliUnixToString(nTime1), milliUnixToString(nTime2)) + log.Infof(context.TODO(), "nTime1: (%d + %d) - (%d + %d) %% %d - %d", cTime, offset, cTime, offset, window, offset) + log.Infof(context.TODO(), "nTime2: (%d + %d) / %d * %d - %d", cTime, offset, window, window, offset) + + assert.Equal(t, nTime1, nTime2) + assert.Equal(t, tc.expected.UnixMilli(), nTime1) }) } } + +// TestTimeZone 时区聚合测试 +func TestTimeZone(t *testing.T) { + timezones := []string{ + "Asia/Shanghai", + //"America/New_York", + //"Pacific/Auckland", + //"Europe/London", + //"UTC", + } + + for _, tz := range timezones { + loc, _ := time.LoadLocation(tz) + _, z := time.Now().In(loc).Zone() + offset := int64(z) * 1000 + window := (time.Hour * 24).Milliseconds() + + fmt.Println("------------------------------------") + fmt.Println(tz, offset/1000/60/60) + fmt.Println("------------------------------------") + + st := time.Date(2025, 3, 18, 0, 0, 0, 0, time.UTC) + for i := 0; i < 50; i++ { + //milli := (st.UnixMilli() + offset) - (st.UnixMilli()+offset)%window - offset + milli := (st.UnixMilli())/window*window - offset + ct := st.In(loc).String() + ot := time.UnixMilli(milli).In(loc).String() + fmt.Println(st, "=>", ct, "=>", ot) + + st = st.Add(time.Hour) + } + fmt.Println("------------------------------------") + } + +} diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index cbb80e46d..85773b8d4 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -11,12 +11,11 @@ package bksql import ( "context" + "encoding/json" "fmt" - "strings" "time" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" @@ -29,6 +28,20 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +const ( + KeyHighLight = "__highlight" + + KeyIndex = "__index" + KeyTableID = "__result_table" + KeyDataLabel = "__data_label" + + TableFieldName = "Field" + TableFieldType = "Type" + + TableTypeVariant = "variant" ) type Instance struct { @@ -48,11 +61,6 @@ type Instance struct { client *Client } -func (i *Instance) QueryReference(ctx context.Context, query *metadata.Query, start int64, end int64) (*prompb.QueryResult, error) { - //TODO implement me - panic("implement me") -} - var _ tsdb.Instance = (*Instance)(nil) type Options struct { @@ -104,14 +112,22 @@ func (i *Instance) checkResult(res *Result) error { return nil } -func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) (*QuerySyncResultData, error) { +func (i *Instance) sqlQuery(ctx context.Context, sql string) (*QuerySyncResultData, error) { var ( data *QuerySyncResultData - ok bool - err error + ok bool + err error + span *trace.Span ) + ctx, span = trace.NewSpan(ctx, "sql-query") + defer span.End(&err) + + if sql == "" { + return data, nil + } + log.Infof(ctx, "%s: %s", i.InstanceType(), sql) span.Set("query-sql", sql) @@ -131,6 +147,7 @@ func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) ( return data, fmt.Errorf("queryAsyncResult type is error: %T", res.Data) } + span.Set("result-size", len(data.List)) return data, nil } @@ -151,123 +168,73 @@ func (i *Instance) dims(dims []string, field string) []string { return dimensions } -func (i *Instance) formatData(ctx context.Context, start time.Time, query *metadata.Query, keys []string, list []map[string]interface{}) (*prompb.QueryResult, error) { - res := &prompb.QueryResult{} +func (i *Instance) getFieldsMap(ctx context.Context, sql string) (map[string]string, error) { + fieldsMap := make(map[string]string) - if len(list) == 0 { - return res, nil - } - // 维度结构体为空则任务异常 - if len(keys) == 0 { - return res, fmt.Errorf("SelectFieldsOrder is empty") + if sql == "" { + return nil, nil } - // 获取该指标的维度 key - dimensions := i.dims(keys, query.Field) - - // 获取 metricLabel - metricLabel := query.MetricLabels(ctx) + data, err := i.sqlQuery(ctx, sql) + if err != nil { + return nil, err + } - tsMap := make(map[string]*prompb.TimeSeries, 0) - for _, d := range list { - // 优先获取时间和值 + for _, list := range data.List { var ( - vt int64 - vv float64 - - vtLong interface{} - vvDouble interface{} - + k string + v string ok bool ) - - if d == nil { + k, ok = list[TableFieldName].(string) + if !ok { continue } - // 获取时间戳，单位是毫秒 - if vtLong, ok = d[timeStamp]; !ok { - vtLong = start.UnixMilli() - } - - if vtLong == nil { + v, ok = list[TableFieldType].(string) + if !ok { continue } - switch vtLong.(type) { - case int64: - vt = vtLong.(int64) - case float64: - vt = int64(vtLong.(float64)) - default: - return res, fmt.Errorf("%s type is error %T, %v", dtEventTimeStamp, vtLong, vtLong) - } - // 获取值 - if vvDouble, ok = d[value]; !ok { - return res, fmt.Errorf("dimension %s is emtpy", value) - } + fieldsMap[k] = v + } - if vvDouble == nil { - continue - } - switch vvDouble.(type) { - case int64: - vv = float64(vvDouble.(int64)) - case float64: - vv = vvDouble.(float64) - default: - return res, fmt.Errorf("%s type is error %T, %v", value, vvDouble, vvDouble) - } + return fieldsMap, nil +} - lbl := make([]prompb.Label, 0) - // 获取维度信息 - for _, dimName := range dimensions { - val, err := getValue(dimName, d) - if err != nil { - return res, fmt.Errorf("dimensions %+v %s", dimensions, err.Error()) - } +func (i *Instance) InitQueryFactory(ctx context.Context, query *metadata.Query, start, end time.Time) (*QueryFactory, error) { + var err error - lbl = append(lbl, prompb.Label{ - Name: dimName, - Value: val, - }) - } + ctx, span := trace.NewSpan(ctx, "instance-init-query-factory") + defer span.End(&err) - // 如果是非时间聚合计算，则无需进行指标名的拼接作用 - if metricLabel != nil { - lbl = append(lbl, *metricLabel) - } + f := NewQueryFactory(ctx, query).WithRangeTime(start, end) - var buf strings.Builder - for _, l := range lbl { - buf.WriteString(l.String()) + // 只有 Doris 才需要获取字段表结构 + if query.Measurement == sqlExpr.Doris { + fieldsMap, err := i.getFieldsMap(ctx, f.DescribeTableSQL()) + if err != nil { + return f, err } - // 同一个 series 进行合并分组 - key := buf.String() - if _, ok := tsMap[key]; !ok { - tsMap[key] = &prompb.TimeSeries{ - Labels: lbl, - Samples: make([]prompb.Sample, 0), + // 只能使用在表结构的字段才能使用 + var keepColumns []string + for _, k := range query.Source { + if _, ok := fieldsMap[k]; ok { + keepColumns = append(keepColumns, k) } } + out, _ := json.Marshal(fieldsMap) + span.Set("table_fields_map", string(out)) - tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ - Value: vv, - Timestamp: vt, - }) + span.Set("keep-columns", keepColumns) + f.WithFieldsMap(fieldsMap).WithKeepColumns(keepColumns) } - // 转换结构体 - res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) - for _, ts := range tsMap { - res.Timeseries = append(res.Timeseries, ts) - } - - return res, nil + return f, nil } -func (i *Instance) table(query *metadata.Query) string { +func (i *Instance) Table(query *metadata.Query) string { table := fmt.Sprintf("`%s`", query.DB) if query.Measurement != "" { table += "." + query.Measurement @@ -276,15 +243,87 @@ func (i *Instance) table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { - return 0, nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, resultTableOptions metadata.ResultTableOptions, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("doris query panic: %s", r) + } + }() + + ctx, span := trace.NewSpan(ctx, "bk-sql-query-raw") + defer span.End(&err) + + span.Set("query-raw-start", start) + span.Set("query-raw-end", end) + + if start.UnixMilli() > end.UnixMilli() || start.UnixMilli() == 0 { + err = fmt.Errorf("start time must less than end time") + return + } + + rangeLeftTime := end.Sub(start) + metric.TsDBRequestRangeMinute(ctx, rangeLeftTime, i.InstanceType()) + + if i.maxLimit > 0 { + maxLimit := i.maxLimit + i.tolerance + // 如果不传 size，则取最大的限制值 + if query.Size == 0 || query.Size > i.maxLimit { + query.Size = maxLimit + } + } + + if len(query.ResultTableOptions) > 0 { + option := query.ResultTableOptions.GetOption(query.TableID, "") + if option != nil { + if option.From != nil { + query.From = *option.From + } + } + } + + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) + if err != nil { + return + } + sql, err := queryFactory.SQL() + if err != nil { + return + } + + data, err := i.sqlQuery(ctx, sql) + if err != nil { + return + } + + if data == nil { + return + } + + span.Set("label-map", queryFactory.GetLabelMap()) + span.Set("data-total-records", data.TotalRecords) + span.Set("data-list-size", len(data.List)) + + for _, list := range data.List { + newData := queryFactory.ReloadListData(list) + newData[KeyIndex] = query.DB + newData[KeyTableID] = query.TableID + newData[KeyDataLabel] = query.DataLabel + + if query.HighLight != nil && query.HighLight.Enable { + newData[KeyHighLight] = queryFactory.HighLight(newData) + } + dataCh <- newData + } + + total = int64(data.TotalRecordSize) + return } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { var ( err error ) - ctx, span := trace.NewSpan(ctx, "bk-sql-raw") + ctx, span := trace.NewSpan(ctx, "bk-sql-query-series-set") defer span.End(&err) span.Set("query-series-set-start", start) @@ -305,14 +344,16 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } } - queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) - + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) + if err != nil { + return storage.ErrSeriesSet(err) + } sql, err := queryFactory.SQL() if err != nil { return storage.ErrSeriesSet(err) } - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return storage.ErrSeriesSet(err) } @@ -328,7 +369,7 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st return storage.ErrSeriesSet(fmt.Errorf("记录数(%d)超过限制(%d)", data.TotalRecords, i.maxLimit)) } - qr, err := i.formatData(ctx, start, query, data.SelectFieldsOrder, data.List) + qr, err := queryFactory.FormatDataToQueryResult(ctx, data.List) if err != nil { return storage.ErrSeriesSet(err) } @@ -337,18 +378,18 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } func (i *Instance) DirectQueryRange(ctx context.Context, promql string, start, end time.Time, step time.Duration) (promql.Matrix, error) { - //TODO implement me - panic("implement me") + log.Warnf(ctx, "%s not support direct query range", i.InstanceType()) + return nil, nil } func (i *Instance) DirectQuery(ctx context.Context, qs string, end time.Time) (promql.Vector, error) { - //TODO implement me - panic("implement me") + log.Warnf(ctx, "%s not support direct query", i.InstanceType()) + return nil, nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { - //TODO implement me - panic("implement me") + log.Warnf(ctx, "%s not support query exemplar", i.InstanceType()) + return nil, nil } func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, start, end time.Time) ([]string, error) { @@ -359,13 +400,20 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s ctx, span := trace.NewSpan(ctx, "bk-sql-label-name") defer span.End(&err) - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 取字段名不需要返回数据，但是 size 不能使用 0，所以还是用 1 + query.Size = 1 + + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) + if err != nil { + return nil, err } - sql := fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", i.table(query), where) - data, err := i.sqlQuery(ctx, sql, span) + + sql, err := queryFactory.SQL() + if err != nil { + return nil, err + } + + data, err := i.sqlQuery(ctx, sql) if err != nil { return nil, err } @@ -388,13 +436,24 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, return nil, fmt.Errorf("not support metric query with %s", name) } - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 使用聚合的方式统计维度组合 + query.Aggregates = metadata.Aggregates{ + { + Dimensions: []string{name}, + Name: "count", + }, + } + + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) + if err != nil { + return nil, err + } + sql, err := queryFactory.SQL() + if err != nil { + return nil, err } - sql := fmt.Sprintf("SELECT COUNT(`%s`) AS `%s`, %s FROM %s WHERE %s GROUP BY %s", query.Field, query.Field, name, i.table(query), where, name) - data, err := i.sqlQuery(ctx, sql, span) + + data, err := i.sqlQuery(ctx, sql) if err != nil { return nil, err } @@ -419,18 +478,15 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, } func (i *Instance) QuerySeries(ctx context.Context, query *metadata.Query, start, end time.Time) ([]map[string]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelNames(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelValues(ctx context.Context, name string, start, end time.Time, limit int, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) InstanceType() string { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 8533f675e..46d9b4f94 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -7,10 +7,11 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -21,28 +22,69 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) -func TestInstance_QueryRaw(t *testing.T) { - - ctx := context.Background() - mock.Init() +func TestInstance_ShowCreateTable(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) - ins, err := NewInstance(ctx, &Options{ - Address: mock.BkSQLUrl, - Timeout: time.Minute, - MaxLimit: 1e4, - Tolerance: 5, - Curl: &curl.HttpCurl{Log: log.DefaultLogger}, + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `132_lol_new_login_queue_login_1min`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":20,"external_api_call_time_mills":{"bkbase_auth_api":64,"bkbase_meta_api":6,"bkbase_apigw_api":25},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c1","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c2","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":27,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":56,"match_query_routing_rule":0,"check_permission":66,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":13168,"timetaken":0.161,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, }) - if err != nil { - assert.Nil(t, err) + + end := time.UnixMilli(1730118889181) + start := time.UnixMilli(1730118589181) + + datasource := "bklog" + db := "132_lol_new_login_queue_login_1min" + measurement := "doris" + + for name, c := range map[string]struct { + query *metadata.Query + expected string + }{ + "field map": { + query: &metadata.Query{ + DataSource: datasource, + DB: db, + Measurement: measurement, + }, + expected: `{"__ext":"varchar(65533)","__shard_key__":"bigint","c1":"text","c2":"text","cloudid":"double","dteventtime":"varchar(32)","dteventtimestamp":"bigint","file":"varchar(65533)","gseindex":"double","iterationindex":"double","level":"varchar(65533)","localtime":"varchar(32)","log":"text","message":"varchar(65533)","path":"text","report_time":"varchar(65533)","serverip":"varchar(65533)","thedate":"int","time":"text","trace_id":"varchar(65533)"}`, + }, + } { + t.Run(name, func(t *testing.T) { + ctx = metadata.InitHashID(ctx) + + fact, err := ins.InitQueryFactory(ctx, c.query, start, end) + assert.Nil(t, err) + + actual, _ := json.Marshal(fact.FieldMap()) + assert.JSONEq(t, c.expected, string(actual)) + + }) } +} + +func TestInstance_QuerySeriesSet(t *testing.T) { + + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND (namespace = 'gz100' OR namespace = 'bgp2\\-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND ((namespace = 'gz100') OR (namespace = 'bgp2-new')) LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", - "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", - "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + // doris + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + + // count by 1m with mysql + "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + + // count by 1m with doris + "SELECT COUNT(`login_rate`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":2,\"_timestamp_\":1730118720000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) end := time.UnixMilli(1730118889181) @@ -53,27 +95,33 @@ func TestInstance_QueryRaw(t *testing.T) { field := "login_rate" tableID := db + ".default" - for idx, c := range []struct { + for name, c := range map[string]struct { query *metadata.Query expected string }{ - { + "count by cloudId with doris": { query: &metadata.Query{ - DataSource: datasource, - TableID: tableID, - DB: db, - MetricName: field, - BkSqlCondition: "namespace = 'gz100' OR namespace = 'bgp2\\-new'", - OffsetInfo: metadata.OffSetInfo{Limit: 10}, + DataSource: datasource, + TableID: tableID, + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "cloudId", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"cloudId"}, + }, + }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":269,"timestamp":1730118660000},{"value":271,"timestamp":1730118480000},{"value":267,"timestamp":1730118540000},{"value":274,"timestamp":1730118600000},{"value":279,"timestamp":1730118420000}],"exemplars":null,"histograms":null}]`, + expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:"},{"name":"cloudId","value":"0"}],"samples":[{"value":6,"timestamp":1730118589181}],"exemplars":null,"histograms":null}]`, }, - { + "count by namespace with mysql": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, DB: db, MetricName: field, + DataLabel: db, Aggregates: metadata.Aggregates{ { Name: "count", @@ -83,12 +131,13 @@ func TestInstance_QueryRaw(t *testing.T) { }, expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"bgp2"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"cq100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn0-new"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn1"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn10"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"nj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"njloadtest"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"pbe"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj101"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null}]`, }, - { + "count with 1m with mysql": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, DB: db, MetricName: field, + DataLabel: db, Aggregates: metadata.Aggregates{ { Name: "count", @@ -96,10 +145,220 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"}],"samples":[{"value":11,"timestamp":1730118600000},{"value":11,"timestamp":1730118660000},{"value":11,"timestamp":1730118720000},{"value":11,"timestamp":1730118780000},{"value":11,"timestamp":1730118840000}],"exemplars":null,"histograms":null}]`, + expected: `[ { + "labels" : [ { + "name" : "__name__", + "value" : "bkdata:132_lol_new_login_queue_login_1min:default:login_rate" + } ], + "samples" : [{ + "value" : 11, + "timestamp" : 1730118600000 + }, { + "value" : 11, + "timestamp" : 1730118660000 + }, { + "value" : 11, + "timestamp" : 1730118720000 + }, { + "value" : 11, + "timestamp" : 1730118780000 + }, { + "value" : 11, + "timestamp" : 1730118840000 + } ], + "exemplars" : null, + "histograms" : null +} ]`, + }, + "count with 1m with doris": { + query: &metadata.Query{ + DataSource: datasource, + TableID: tableID, + DB: "2_bklog_bkunify_query_doris", + Measurement: sqlExpr.Doris, + MetricName: field, + DataLabel: db, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Minute, + }, + }, + }, + expected: `[ { + "labels" : [ { + "name" : "__name__", + "value" : "bkdata:132_lol_new_login_queue_login_1min:default:login_rate" + } ], + "samples" : [ { + "timestamp" : 1730118540000 + }, { + "timestamp" : 1730118600000 + }, { + "timestamp" : 1730118660000 + }, { + "value" : 2, + "timestamp" : 1730118720000 + }, { + "timestamp" : 1730118780000 + }, { + "timestamp" : 1730118840000 + } ], + "exemplars" : null, + "histograms" : null +} ]`, + }, + } { + t.Run(name, func(t *testing.T) { + ctx = metadata.InitHashID(ctx) + if c.query.DB == "" { + c.query.DB = db + } + if c.query.Field == "" { + c.query.Field = field + } + + set := ins.QuerySeriesSet(ctx, c.query, start, end) + ts, err := mock.SeriesSetToTimeSeries(set) + assert.Nil(t, err) + + actual, err := json.Marshal(ts) + assert.Nil(t, err) + + fmt.Println(string(actual)) + + assert.JSONEq(t, c.expected, string(actual)) + }) + } +} + +func TestInstance_QueryRaw(t *testing.T) { + + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) + + mock.BkSQL.Set(map[string]any{ + // query raw by doris use condition in highlight + "SHOW CREATE TABLE `2_bklog_pure_v4_log_doris_for_unify_query`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":69,"bkbase_meta_api":9,"bkbase_apigw_api":25},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"__shard_key__","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"cloudId","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"serverIp","Type":"varchar(512)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"path","Type":"varchar(512)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"gseIndex","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"iterationIndex","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"dtEventTimeStamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dtEventTime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localTime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":6,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":66,"match_query_routing_rule":0,"check_permission":69,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_pure_v4_log_doris_for_unify_query_2","total_record_size":11808,"timetaken":0.148,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_pure_v4_log_doris_for_unify_query"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + // query raw by doris use condition in highlight + "SELECT *, `dtEventTimeStamp` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `2_bklog_pure_v4_log_doris_for_unify_query`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND (`message` MATCH_PHRASE_PREFIX 'Bk-Query-Source' OR (`level` MATCH_PHRASE_PREFIX 'error' OR `level` MATCH_PHRASE_PREFIX 'info')) LIMIT 5": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_pure_v4_log_doris_for_unify_query":{"start":"2025042100","end":"2025042123"}},"cluster":"doris-test","totalRecords":5,"external_api_call_time_mills":{"bkbase_auth_api":38,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":0.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.643Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.643Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":2.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.790Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.790Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":4.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.855Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.855Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":6.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:01.030Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:01.030Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":8.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:305","level":"info","log":"2025-04-21T11:25:01.199Z\tinfo\thttp/handler.go:305\t[adb84ecc380008245cdb800b6fd54d7f] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","report_time":"2025-04-21T11:25:01.199Z","time":"1745234704","trace_id":"adb84ecc380008245cdb800b6fd54d7f","_value_":1745234704000,"_timestamp_":1745234704000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":19,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":41,"match_query_routing_rule":0,"check_permission":39,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["thedate","__shard_key__","cloudId","serverIp","path","gseIndex","iterationIndex","dtEventTimeStamp","dtEventTime","localTime","__ext","file","level","log","message","report_time","time","trace_id","_value_","_timestamp_"],"total_record_size":30880,"timetaken":0.104,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"__shard_key__","field_index":1},{"field_type":"double","field_name":"__c2","field_alias":"cloudId","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"serverIp","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"path","field_index":4},{"field_type":"double","field_name":"__c5","field_alias":"gseIndex","field_index":5},{"field_type":"double","field_name":"__c6","field_alias":"iterationIndex","field_index":6},{"field_type":"long","field_name":"__c7","field_alias":"dtEventTimeStamp","field_index":7},{"field_type":"string","field_name":"__c8","field_alias":"dtEventTime","field_index":8},{"field_type":"string","field_name":"__c9","field_alias":"localTime","field_index":9},{"field_type":"string","field_name":"__c10","field_alias":"__ext","field_index":10},{"field_type":"string","field_name":"__c11","field_alias":"file","field_index":11},{"field_type":"string","field_name":"__c12","field_alias":"level","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"string","field_name":"__c14","field_alias":"message","field_index":14},{"field_type":"string","field_name":"__c15","field_alias":"report_time","field_index":15},{"field_type":"string","field_name":"__c16","field_alias":"time","field_index":16},{"field_type":"string","field_name":"__c17","field_alias":"trace_id","field_index":17},{"field_type":"long","field_name":"__c18","field_alias":"_value_","field_index":18},{"field_type":"long","field_name":"__c19","field_alias":"_timestamp_","field_index":19}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_pure_v4_log_doris_for_unify_query"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + + // query with in + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + + // query raw by doris + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + + // query raw by doris and highlight + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND (`log` MATCH_PHRASE_PREFIX 'metricbeat_scrape' OR `log` MATCH_PHRASE_PREFIX 'metricbeat') LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + }) + + end := time.UnixMilli(1730118889181) + start := time.UnixMilli(1730118589181) + + datasource := "bkdata" + db := "132_lol_new_login_queue_login_1min" + field := "login_rate" + tableID := db + ".default" + + for name, c := range map[string]struct { + query *metadata.Query + expected string + }{ + "query with in": { + query: &metadata.Query{ + DataSource: datasource, + TableID: tableID, + DB: db, + DataLabel: db, + MetricName: field, + BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", + OffsetInfo: metadata.OffSetInfo{Limit: 10}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "namespace", + Operator: metadata.ConditionEqual, + Value: []string{"gz100", "bgp2-new"}, + }, + }, + }, + }, + expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:32:00","_startTime_":"2024-10-28 20:31:00","_timestamp_":1730118660000,"_value_":269,"dtEventTime":"2024-10-28 20:31:00","dtEventTimeStamp":1730118660000,"localTime":"2024-10-28 20:32:03","login_rate":269,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:29:00","_startTime_":"2024-10-28 20:28:00","_timestamp_":1730118480000,"_value_":271,"dtEventTime":"2024-10-28 20:28:00","dtEventTimeStamp":1730118480000,"localTime":"2024-10-28 20:29:03","login_rate":271,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:30:00","_startTime_":"2024-10-28 20:29:00","_timestamp_":1730118540000,"_value_":267,"dtEventTime":"2024-10-28 20:29:00","dtEventTimeStamp":1730118540000,"localTime":"2024-10-28 20:30:02","login_rate":267,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:31:00","_startTime_":"2024-10-28 20:30:00","_timestamp_":1730118600000,"_value_":274,"dtEventTime":"2024-10-28 20:30:00","dtEventTimeStamp":1730118600000,"localTime":"2024-10-28 20:31:04","login_rate":274,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:28:00","_startTime_":"2024-10-28 20:27:00","_timestamp_":1730118420000,"_value_":279,"dtEventTime":"2024-10-28 20:27:00","dtEventTimeStamp":1730118420000,"localTime":"2024-10-28 20:28:03","login_rate":279,"namespace":"gz100","thedate":20241028}]`, + }, + "query raw by doris": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + Size: 2, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, + "query raw by doris use querystring in highlight": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + QueryString: "metricbeat_scrape metricbeat", + Size: 2, + HighLight: &metadata.HighLight{ + MaxAnalyzedOffset: 0, + Enable: true, + }, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, + "query raw by doris use querystring in highlight and max analyzed offset is 40": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + QueryString: "metricbeat_scrape metricbeat", + Size: 2, + HighLight: &metadata.HighLight{ + MaxAnalyzedOffset: 40, + Enable: true, + }, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, + "query raw by doris use condition in highlight": { + query: &metadata.Query{ + TableID: "2_bklog.bklog_pure_v4_log_doris_for_unify_query", + DB: "2_bklog_pure_v4_log_doris_for_unify_query", + Measurement: "doris", + MetricName: "dtEventTimeStamp", + Field: "dtEventTimeStamp", + DataLabel: "log_index_set_1183", + AllConditions: metadata.AllConditions{ + { + {DimensionName: "message", Value: []string{"Bk-Query-Source"}, Operator: "contains"}, + }, + { + {DimensionName: "level", Value: []string{"error", "info"}, Operator: "contains"}, + }, + }, + Size: 5, + HighLight: &metadata.HighLight{ + Enable: true, + }, + }, + expected: `[{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":0,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.643Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.643Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":2,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.790Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.790Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":4,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.855Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.855Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":6,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:01.030Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:01.030Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:305","gseIndex":4281730,"iterationIndex":8,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:01.199Z\tinfo\thttp/handler.go:305\t[adb84ecc380008245cdb800b6fd54d7f] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:01.199Z","thedate":20250421,"time":"1745234704","trace_id":"adb84ecc380008245cdb800b6fd54d7f"}]`, }, } { - t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) if c.query.DB == "" { c.query.DB = db @@ -107,11 +366,27 @@ func TestInstance_QueryRaw(t *testing.T) { if c.query.Field == "" { c.query.Field = field } - ss := ins.QuerySeriesSet(ctx, c.query, start, end) - timeSeries, err := mock.SeriesSetToTimeSeries(ss) + dataCh := make(chan map[string]any) + + go func() { + defer func() { + close(dataCh) + }() + + _, _, err := ins.QueryRawData(ctx, c.query, start, end, dataCh) + assert.Nil(t, err) + }() + + list := make([]map[string]any, 0) + for d := range dataCh { + list = append(list, d) + } + + actual, err := json.Marshal(list) assert.Nil(t, err) - assert.Equal(t, c.expected, timeSeries.String()) + + assert.JSONEq(t, c.expected, string(actual)) }) } } @@ -123,6 +398,7 @@ func TestInstance_bkSql(t *testing.T) { end := time.UnixMilli(1718193555000) testCases := []struct { + name string start time.Time end time.Time query *metadata.Query @@ -130,10 +406,11 @@ func TestInstance_bkSql(t *testing.T) { expected string }{ { + name: "namespace in and aggregate count", query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", - BkSqlCondition: "namespace REGEXP '^(bgp2\\-new|gz100)$'", + BkSqlCondition: "`namespace` IN ('bgp2-new', 'gz100')", Aggregates: metadata.Aggregates{ { Name: "count", @@ -141,149 +418,412 @@ func TestInstance_bkSql(t *testing.T) { Window: time.Second * 15, }, }, - }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (namespace REGEXP '^(bgp2\\-new|gz100)$') GROUP BY `namespace`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0)) ORDER BY `_timestamp_` ASC", - }, - { - query: &metadata.Query{ - DB: "132_hander_opmon_avg", - Field: "value", - Aggregates: metadata.Aggregates{ - { - Name: "sum", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"bgp2-new", "gz100"}, + }, }, }, }, - - expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 15000 * 15000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (dtEventTimeStamp + 0) / 15000 * 15000 - 0 ORDER BY `_timestamp_` ASC", }, { + name: "conditions with or", query: &metadata.Query{ - DB: "100133_ieod_logsearch4_errorlog_p", - Measurement: "doris", - Field: "value", - Size: 5, + DB: "132_lol_new_login_queue_login_1min", + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, + }, + }, + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' LIMIT 5", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, { + name: "conditions with or and", query: &metadata.Query{ - DB: "100133_ieod_logsearch4_errorlog_p", - Measurement: "doris", - Field: "value", - Orders: metadata.Orders{ - { - Name: "_time", - Ast: false, + DB: "132_lol_new_login_queue_login_1min", + Measurement: sqlExpr.Doris, + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, }, }, + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `_timestamp_` DESC", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, { + name: "conditions with or and like", query: &metadata.Query{ - DB: "100133_ieod_logsearch4_errorlog_p", - Measurement: "doris", - Field: "gseIndex", - Aggregates: metadata.Aggregates{ - { - Name: "count", - Dimensions: []string{ - "ip", + DB: "132_lol_new_login_queue_login_1min", + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test"}, }, }, }, - Size: 5, + BkSqlCondition: "(`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, - - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY `ip` LIMIT 5", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, { - start: time.Unix(1733756400, 0), - end: time.Unix(1733846399, 0), + name: "aggregate sum", query: &metadata.Query{ - DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", - Field: "matchstep_start_to_fail_0_100", + DB: "132_hander_opmon_avg", + Field: "value", Aggregates: metadata.Aggregates{ { - Name: "count", + Name: "sum", }, }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { - start: time.Unix(1733756400, 0), - end: time.Unix(1733846399, 0), + name: "aggregate cardinality with mysql", query: &metadata.Query{ - DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", - Field: "matchstep_start_to_fail_0_100", + DB: "2_bklog_bkunify_query_doris", + Measurement: "", + Field: "gseIndex", Aggregates: metadata.Aggregates{ { - Name: "count", - Window: time.Hour, + Name: "cardinality", + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, }, }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { - start: time.Unix(1733756400, 0), - end: time.Unix(1733846399, 0), + name: "aggregate date_histogram with mysql", query: &metadata.Query{ - DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", - Field: "matchstep_start_to_fail_0_100", + DB: "2_bklog_bkunify_query_doris", + Measurement: "", + Field: "gseIndex", Aggregates: metadata.Aggregates{ { - Name: "count", - Window: time.Hour * 24, - TimeZone: "Asia/Shanghai", + Name: "count", + }, + { + Name: "date_histogram", + Window: time.Minute * 5, + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, }, }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 86400000 - 28800000))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 86400000 - 28800000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 300000 * 300000 - 0) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY (dtEventTimeStamp + 0) / 300000 * 300000 - 0 ORDER BY `_timestamp_` ASC", }, { - start: time.Unix(1733756400, 0), - end: time.Unix(1733846399, 0), + name: "aggregate cardinality with doris", query: &metadata.Query{ - DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", - Field: "matchstep_start_to_fail_0_100", + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "gseIndex", Aggregates: metadata.Aggregates{ { - Name: "count", - Window: time.Hour * 24 * 7, - TimeZone: "Asia/Shanghai", + Name: "cardinality", + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, }, }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 604800000 - 28800000))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 28800000) % 604800000 - 28800000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { - start: time.Unix(1733756400, 0), - end: time.Unix(1733846399, 0), + name: "aggregate date_histogram with doris", query: &metadata.Query{ - DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", - Field: "matchstep_start_to_fail_0_100", + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "gseIndex", Aggregates: metadata.Aggregates{ { - Name: "count", - Window: time.Hour*24*2 + time.Hour*2, - TimeZone: "Asia/Shanghai", + Name: "count", + }, + { + Name: "date_histogram", + Window: time.Minute * 5, + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, }, }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 180000000 - 0))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 180000000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + + //{ + // name: "aggregate multi function", + // query: &metadata.Query{ + // DB: "132_hander_opmon_avg", + // Field: "value", + // Aggregates: metadata.Aggregates{ + // { + // Name: "sum", + // Field: "value", + // Window: time.Second * 15, + // }, + // { + // Name: "count", + // Field: "other", + // Window: time.Hour, + // }, + // }, + // }, + // + // // TODO 适配多字段查询特性 + // expected: "", + //}, + { + name: "query raw order ", + query: &metadata.Query{ + DB: "100133_ieod_logsearch4_errorlog_p", + Measurement: "doris", + Field: "value", + Size: 5, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `dtEventTimeStamp` DESC, `gseIndex` DESC, `iterationIndex` DESC LIMIT 5", + }, + { + name: "query raw", + query: &metadata.Query{ + DB: "100133_ieod_logsearch4_errorlog_p", + Measurement: "doris", + Field: "value", + Size: 5, + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' LIMIT 5", + }, + { + name: "query raw with order desc", + query: &metadata.Query{ + DB: "100133_ieod_logsearch4_errorlog_p", + Measurement: "doris", + Field: "value", + Orders: metadata.Orders{ + { + Name: "_time", + Ast: false, + }, + }, + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `_timestamp_` DESC", + }, + { + name: "query aggregate count and dimensions", + query: &metadata.Query{ + DB: "100133_ieod_logsearch4_errorlog_p", + Measurement: "doris", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{ + "ip", + }, + }, + }, + Size: 5, + }, + + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY `ip` LIMIT 5", + }, + { + name: "query aggregate count", + start: time.Unix(1733756400, 0), + end: time.Unix(1733846399, 0), + query: &metadata.Query{ + DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", + Field: "matchstep_start_to_fail_0_100", + Aggregates: metadata.Aggregates{ + { + Name: "count", + }, + }, + }, + + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + }, + { + name: "query aggregate count with window hour", + start: time.Unix(1733756400, 0), + end: time.Unix(1733846399, 0), + query: &metadata.Query{ + DB: "101068_MatchFullLinkTimeConsumptionFlow_CostTime", + Field: "matchstep_start_to_fail_0_100", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour, + }, + }, + }, + + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 3600000 * 3600000 - 0) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (dtEventTimeStamp + 0) / 3600000 * 3600000 - 0 ORDER BY `_timestamp_` ASC", }, } - for i, c := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + for _, c := range testCases { + t.Run(c.name, func(t *testing.T) { ctx := metadata.InitHashID(context.Background()) if c.start.Unix() <= 0 { c.start = start @@ -292,9 +832,513 @@ func TestInstance_bkSql(t *testing.T) { c.end = end } - sql, err := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end).SQL() + fieldsMap := map[string]string{ + "text": sqlExpr.DorisTypeText, + } + + condition, err := sqlExpr.NewSQLExpr(c.query.Measurement).WithFieldsMap(fieldsMap).ParserAllConditions(c.query.AllConditions) + assert.Nil(t, err) + if err == nil { + assert.Equal(t, c.query.BkSqlCondition, condition) + } + + fact := bksql.NewQueryFactory(ctx, c.query).WithFieldsMap(fieldsMap).WithRangeTime(c.start, c.end) + sql, err := fact.SQL() assert.Nil(t, err) assert.Equal(t, c.expected, sql) }) } } + +func TestInstance_bkSql_EdgeCases(t *testing.T) { + mock.Init() + + // 基础时间范围 + baseStart := time.UnixMilli(1718189940000) + baseEnd := time.UnixMilli(1718193555000) + + // 跨天时间范围 + crossDayStart := time.Unix(1733756400, 0) // 2024-12-09 00:00:00 + crossDayEnd := time.Unix(1733846399, 0) // 2024-12-09 23:59:59 + + testCases := []struct { + name string + start time.Time + end time.Time + query *metadata.Query + expected string + err error + }{ + // 测试用例1: 无聚合函数的原始查询 + { + name: "mysql raw query without aggregation", + query: &metadata.Query{ + DB: "test_db", + Field: "value", + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `test_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例2: 多聚合函数组合 + { + name: "mysql multiple aggregates", + query: &metadata.Query{ + DB: "metrics_db", + Field: "temperature", + Aggregates: metadata.Aggregates{ + {Name: "max"}, + {Name: "min"}, + }, + }, + expected: "SELECT MAX(`temperature`) AS `_value_`, MIN(`temperature`) AS `_value_` FROM `metrics_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例3: 复杂条件组合 + { + name: "mysql complex conditions", + query: &metadata.Query{ + DB: "security_logs", + Field: "duration", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "severity", + Operator: metadata.ConditionEqual, + Value: []string{"high", "critical"}, + }, + { + DimensionName: "source_ip", + Operator: metadata.ConditionNotContains, + Value: []string{"192.168.1.1"}, + }, + }, + }, + }, + expected: "SELECT *, `duration` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `security_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND `severity` IN ('high', 'critical') AND `source_ip` != '192.168.1.1'", + }, + + // 测试用例4: 多字段排序 + { + name: "mysql multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "amount", + Orders: metadata.Orders{ + { + Name: "timestamp", + Ast: true, + }, + { + Name: "account_id", + }, + }, + }, + expected: "SELECT *, `amount` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `account_id` DESC, `timestamp` ASC", + }, + + // 测试用例5: 特殊字符转义 + { + name: "mysql special characters in fields", + query: &metadata.Query{ + DB: "special_metrics", + Measurement: "select", // 保留字作为measurement + Field: "*", + Aggregates: metadata.Aggregates{ + {Name: "sum"}, + }, + }, + expected: "SELECT SUM(`*`) AS `_value_` FROM `special_metrics`.select WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例6: 零窗口时间 + { + name: "mysql zero window size", + query: &metadata.Query{ + DB: "time_series_data", + Field: "value", + Aggregates: metadata.Aggregates{ + { + Name: "avg", + Window: 0, + }, + }, + }, + expected: "SELECT AVG(`value`) AS `_value_` FROM `time_series_data` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例7: 跨多天的时间范围 + { + name: "mysql multi-day time range", + start: crossDayStart, + end: crossDayEnd, + query: &metadata.Query{ + DB: "daily_metrics", + Field: "active_users", + Aggregates: metadata.Aggregates{ + {Name: "count"}, + }, + }, + expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + }, + + // 测试用例8: 默认处理 object 字段 + { + name: "mysql default multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "__ext.container_id", + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.container_id", + Operator: metadata.ConditionEqual, + Value: []string{"1234567890"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.container_id", "test"}, + }, + }, + Orders: metadata.Orders{ + { + Name: "timestamp", + Ast: true, + }, + { + Name: "__ext.container_id", + }, + }, + }, + err: fmt.Errorf("query is not support object with __ext.container_id"), + }, + + // 测试用例9: doris 处理 object 字段 + { + name: "doris default multiple order fields", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.io_kubernetes_workload_name", + Operator: metadata.ConditionEqual, + Value: []string{"bkm-daemonset-worker"}, + }, + { + DimensionName: "bk_host_id", + Operator: metadata.ConditionEqual, + Value: []string{"267730"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type"}, + }, + }, + Orders: metadata.Orders{ + { + Name: "__ext.io_kubernetes_workload_name", + }, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY __ext__bk_46__io_kubernetes_workload_name, __ext__bk_46__io_kubernetes_workload_type ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", + }, + // 测试用例10: doris 处理 object 字段 + 时间聚合 + { + name: "doris default multiple order fields and time aggregate", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.io_kubernetes_workload_name", + Operator: metadata.ConditionEqual, + Value: []string{"bkm-daemonset-worker"}, + }, + { + DimensionName: "bk_host_id", + Operator: metadata.ConditionEqual, + Value: []string{"267730"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type"}, + Window: time.Minute, + }, + }, + Orders: metadata.Orders{ + { + Name: "__ext.io_kubernetes_workload_name", + }, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY __ext__bk_46__io_kubernetes_workload_name, __ext__bk_46__io_kubernetes_workload_type, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + }, + // 测试用例11: doris 处理 object 字段 + 时间聚合 5m + { + name: "doris default multiple order fields and time aggregate 5m", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{}, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name"}, + Window: time.Minute * 5, + }, + }, + Orders: metadata.Orders{ + { + Name: "__ext.io_kubernetes_workload_name", + }, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY __ext__bk_46__io_kubernetes_workload_name, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + }, + // 测试用例12: doris 处理 object 字段 + 时间聚合 15s + { + name: "doris default multiple order fields and time aggregate 15s", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{}, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name"}, + Window: time.Second * 15, + }, + }, + Orders: metadata.Orders{ + { + Name: "__ext.io_kubernetes_workload_name", + }, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY __ext__bk_46__io_kubernetes_workload_name, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + }, + // 测试用例13: doris 处理多层级 object 字段 + { + name: "doris default multiple order fields and time aggregate 1m", + query: &metadata.Query{ + DB: "2_bkapm_trace_bkop_doris", + Measurement: sqlExpr.Doris, + Field: "attributes.http.host", + Size: 1, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "attributes.http.host", + Operator: metadata.ConditionNotEqual, + Value: []string{""}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"attributes.http.host"}, + Window: time.Minute, + }, + }, + Orders: metadata.Orders{ + { + Name: "attributes.http.host", + }, + }, + }, + start: time.UnixMilli(1744880448784), + end: time.UnixMilli(1744884048785), + expected: "SELECT CAST(attributes[\"http.host\"] AS STRING) AS `attributes__bk_46__http__bk_46__host`, COUNT(CAST(attributes[\"http.host\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bkapm_trace_bkop_doris`.doris WHERE `dtEventTimeStamp` >= 1744880448784 AND `dtEventTimeStamp` <= 1744884048785 AND `thedate` = '20250417' AND CAST(attributes[\"http.host\"] AS STRING) IS NOT NULL GROUP BY attributes__bk_46__http__bk_46__host, _timestamp_ ORDER BY CAST(attributes[\"http.host\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 1", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + + // 设置默认时间范围 + start := tc.start + if start.IsZero() { + start = baseStart + } + end := tc.end + if end.IsZero() { + end = baseEnd + } + + // SQL生成验证 + fact := bksql.NewQueryFactory(ctx, tc.query).WithFieldsMap(map[string]string{ + "text": sqlExpr.DorisTypeText, + }).WithRangeTime(start, end) + generatedSQL, err := fact.SQL() + + if tc.err != nil { + assert.Equal(t, tc.err, err) + } else { + assert.Nil(t, err) + if err == nil { + assert.Equal(t, tc.expected, generatedSQL) + + // 验证时间条件 + if tc.start.IsZero() && tc.end.IsZero() { + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` <= %d", baseEnd.UnixMilli())) + } + } + } + }) + } +} + +// 测试正常标签名查询 +func TestInstance_QueryLabelNames_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` <= 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + }, + expectedNames: []string{ + "dteventtimestamp", "dteventtime", "localtime", "_starttime_", "_endtime_", "bk_host_id", "__ext", "cloudid", "path", "gseindex", "iterationindex", "log", "time", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelNames(ctx, tt.qry, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 测试正常标签名查询 +func TestInstance_QueryLabelValues_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` <= 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + key string + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Size: 2, + }, + key: "bk_host_id", + expectedNames: []string{ + "5843771", "4580470", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelValues(ctx, tt.qry, tt.key, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 创建测试用Instance +func createTestInstance(ctx context.Context) *bksql.Instance { + mock.Init() + + ins, err := bksql.NewInstance(ctx, &bksql.Options{ + Address: mock.BkBaseUrl, + Timeout: time.Minute, + MaxLimit: 1e4, + Tolerance: 5, + Curl: &curl.HttpCurl{Log: log.DefaultLogger}, + }) + if err != nil { + log.Fatalf(ctx, err.Error()) + return nil + } + return ins +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go new file mode 100644 index 000000000..a271b82ee --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -0,0 +1,435 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr + +import ( + "fmt" + "strings" + "time" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" +) + +const ( + SelectAll = "*" + TimeStamp = "_timestamp_" + Value = "_value_" + + FieldValue = "_value" + FieldTime = "_time" + + theDate = "thedate" +) + +// ErrorMatchAll 定义全字段检索错误提示信息 +var ( + ErrorMatchAll = "不支持全字段检索" +) + +type TimeAggregate struct { + Window time.Duration + OffsetMillis int64 +} + +// SQLExpr 定义SQL表达式生成接口 +// 接口包含字段映射、查询字符串解析、全条件解析等功能 +type SQLExpr interface { + // WithKeepColumns 设置保留字段 + WithKeepColumns([]string) SQLExpr + // WithFieldsMap 设置字段类型 + WithFieldsMap(fieldsMap map[string]string) SQLExpr + // WithEncode 字段转换方法 + WithEncode(func(string) string) SQLExpr + // WithInternalFields 设置内部字段 + WithInternalFields(timeField, valueField string) SQLExpr + // IsSetLabels 是否保留查询标签 + IsSetLabels(bool) SQLExpr + // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 + ParserQueryString(qs string) (string, error) + // ParserAllConditions 解析全量条件生成SQL条件表达式 + ParserAllConditions(allConditions metadata.AllConditions) (string, error) + // ParserAggregatesAndOrders 解析聚合条件生成SQL条件表达式 + ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, TimeAggregate, error) + // DescribeTableSQL 返回当前表结构 + DescribeTableSQL(table string) string + // FieldMap 返回当前表结构 + FieldMap() map[string]string + // GetLabelMap 返回当前查询值 + GetLabelMap() map[string][]string + // Type 返回表达式类型 + Type() string +} + +// SQL表达式注册管理相关变量 +var ( + _ SQLExpr = (*DefaultSQLExpr)(nil) // 接口实现检查 +) + +// NewSQLExpr 获取指定key的SQL表达式实现 +// 参数： +// +// key - 注册时使用的标识符 +// +// 返回值： +// +// 找到返回对应实现，未找到返回默认实现 +func NewSQLExpr(key string) SQLExpr { + switch key { + case Doris: + return &DorisSQLExpr{} + default: + return &DefaultSQLExpr{} + } +} + +// DefaultSQLExpr SQL表达式默认实现 +type DefaultSQLExpr struct { + encodeFunc func(string) string + + keepColumns []string + fieldMap map[string]string + + timeField string + valueField string +} + +func (d *DefaultSQLExpr) Type() string { + return "default" +} + +func (d *DefaultSQLExpr) IsSetLabels(_ bool) SQLExpr { + return d +} + +func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { + d.timeField = timeField + d.valueField = valueField + return d +} + +func (d *DefaultSQLExpr) WithEncode(fn func(string) string) SQLExpr { + d.encodeFunc = fn + return d +} + +func (d *DefaultSQLExpr) WithFieldsMap(fieldMap map[string]string) SQLExpr { + d.fieldMap = fieldMap + return d +} + +func (d *DefaultSQLExpr) WithKeepColumns(cols []string) SQLExpr { + d.keepColumns = cols + return d +} + +func (d *DefaultSQLExpr) GetLabelMap() map[string][]string { + return nil +} + +func (d *DefaultSQLExpr) FieldMap() map[string]string { + return d.fieldMap +} + +// ParserQueryString 解析查询字符串（当前实现返回空） +func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { + return "", nil +} + +// ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 +func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, timeAggregate TimeAggregate, err error) { + valueField, err := d.dimTransform(d.valueField) + if err != nil { + return + } + + var ( + window time.Duration + offsetMillis int64 + timezone string + dimensionMap = make(map[string]struct{}) + ) + for _, agg := range aggregates { + for _, dim := range agg.Dimensions { + dim, err = d.dimTransform(dim) + if err != nil { + return + } + dimensionMap[dim] = struct{}{} + + selectFields = append(selectFields, dim) + groupByFields = append(groupByFields, dim) + } + + if valueField == "" { + valueField = SelectAll + } + + switch agg.Name { + case "cardinality": + selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + // date_histogram 不支持无需进行函数聚合 + case "date_histogram": + default: + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + } + + if agg.Window > 0 { + window = agg.Window + timezone = agg.TimeZone + } + } + + if window > 0 { + if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(timezone) + if locErr != nil { + loc = time.UTC + } + // 获取时区偏移量 + _, offset := time.Now().In(loc).Zone() + offsetMillis = int64(offset) * 1e3 + } + + timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, offsetMillis, window.Milliseconds(), window.Milliseconds(), offsetMillis) + + groupByFields = append(groupByFields, timeField) + selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) + } + + if len(selectFields) == 0 { + selectFields = append(selectFields, SelectAll) + if valueField != "" { + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, Value)) + } + if d.timeField != "" { + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, TimeStamp)) + } + } + + for _, order := range orders { + // 如果是聚合操作的话，只能使用维度进行排序 + if len(aggregates) > 0 { + if _, ok := dimensionMap[order.Name]; !ok { + continue + } + } + + var orderField string + switch order.Name { + case FieldValue: + orderField = d.valueField + case FieldTime: + orderField = TimeStamp + default: + orderField = order.Name + } + + orderField, err = d.dimTransform(orderField) + if err != nil { + return + } + + ascName := "ASC" + if !order.Ast { + ascName = "DESC" + } + orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) + } + + // 回传时间聚合信息 + timeAggregate = TimeAggregate{ + Window: window, + OffsetMillis: offsetMillis, + } + + return +} + +// ParserAllConditions 解析全量条件生成SQL条件表达式 +// 实现逻辑： +// 1. 将多个AND条件组合成OR条件 +// 2. 当有多个OR条件时用括号包裹 +func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + var ( + orConditions []string + ) + + // 遍历所有OR条件组 + for _, conditions := range allConditions { + var andConditions []string + // 处理每个AND条件组 + for _, cond := range conditions { + buildCondition, err := d.buildCondition(cond) + if err != nil { + return "", err + } + if buildCondition != "" { + andConditions = append(andConditions, buildCondition) + } + } + // 合并AND条件 + if len(andConditions) > 0 { + orConditions = append(orConditions, strings.Join(andConditions, " AND ")) + } + } + + // 处理最终OR条件组合 + if len(orConditions) > 0 { + if len(orConditions) == 1 { + return orConditions[0], nil + } + return fmt.Sprintf("(%s)", strings.Join(orConditions, " OR ")), nil + } + + return "", nil +} + +func (d *DefaultSQLExpr) DescribeTableSQL(table string) string { + return "" +} + +// buildCondition 构建单个条件表达式 +func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { + if len(c.Value) == 0 { + return "", nil + } + + var ( + key string + op string + val string + ) + + key, err := d.dimTransform(c.DimensionName) + if err != nil { + return "", err + } + + // 对值进行转义处理 + for i, v := range c.Value { + c.Value[i] = d.valueTransform(v) + } + + // 根据操作符类型生成不同的SQL表达式 + switch c.Operator { + // 处理等于类操作符（=, IN, LIKE） + case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) > 1 && !c.IsWildcard { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "LIKE" + } else { + format = "'%s'" + op = "=" + } + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) + } + } + // 处理不等于类操作符（!=, NOT IN, NOT LIKE） + case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) > 1 && !c.IsWildcard { + op = "NOT IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "NOT LIKE" + } else { + format = "'%s'" + op = "!=" + } + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) + } + } + // 处理正则表达式匹配 + case metadata.ConditionRegEqual: + op = "REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) // 多个值用|连接 + case metadata.ConditionNotRegEqual: + op = "NOT REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) + // 处理数值比较操作符（>, >=, <, <=） + case metadata.ConditionGt: + op = ">" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionGte: + op = ">=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLt: + op = "<" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLte: + op = "<=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + default: + return "", fmt.Errorf("unknown operator %s", c.Operator) + } + + if key != "" { + return fmt.Sprintf("%s %s %s", key, op, val), nil + } + return val, nil +} + +func (d *DefaultSQLExpr) valueTransform(s string) string { + if strings.Contains(s, "'") { + s = strings.ReplaceAll(s, "'", "''") + } + return s +} + +func (d *DefaultSQLExpr) dimTransform(s string) (string, error) { + if s == "" { + return "", nil + } + fs := strings.Split(s, ".") + if len(fs) > 1 { + return "", fmt.Errorf("query is not support object with %s", s) + } + + return fmt.Sprintf("`%s`", s), nil +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go new file mode 100644 index 000000000..fdd201770 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go @@ -0,0 +1,139 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +// TestParserAllConditions 测试全条件解析的主测试函数 +func TestParserAllConditions(t *testing.T) { + d := sqlExpr.NewSQLExpr("") + + t.Run("空条件测试", func(t *testing.T) { + conditions := metadata.AllConditions{} + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "", result) + }) + + t.Run("单OR组单AND条件", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "host", Operator: metadata.ConditionEqual, Value: []string{"server1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`host` = 'server1'", result) + }) + + t.Run("多OR组组合", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "os", Operator: metadata.ConditionEqual, Value: []string{"linux"}}, + {DimensionName: "status", Operator: metadata.ConditionGt, Value: []string{"5"}}, + }, + { + {DimensionName: "region", Operator: metadata.ConditionEqual, Value: []string{"north", "south"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`os` = 'linux' AND `status` > 5 OR `region` IN ('north', 'south'))" + assert.Equal(t, expected, result) + }) + + t.Run("包含空AND条件过滤", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "log", Operator: metadata.ConditionRegEqual, Value: []string{}}, // 空值条件 + {DimensionName: "level", Operator: metadata.ConditionNotEqual, Value: []string{"debug"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`level` != 'debug'", result) + }) + + t.Run("混合操作符测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "cpu", Operator: metadata.ConditionGte, Value: []string{"80"}}, + {DimensionName: "memory", Operator: metadata.ConditionLt, Value: []string{"90"}}, + }, + { + {DimensionName: "service", Operator: metadata.ConditionContains, Value: []string{"api", "db"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`cpu` >= 80 AND `memory` < 90 OR `service` IN ('api', 'db'))" + assert.Equal(t, expected, result) + }) + + t.Run("错误条件传递测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "error", Operator: metadata.ConditionGt, Value: []string{"1", "2"}}, + }, + } + _, err := d.ParserAllConditions(conditions) + assert.Error(t, err) + assert.Contains(t, err.Error(), "operator > only support 1 value") + }) + + t.Run("多层嵌套组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "A", Operator: metadata.ConditionEqual, Value: []string{"1"}}, + {DimensionName: "B", Operator: metadata.ConditionNotEqual, Value: []string{"2"}}, + }, + { + {DimensionName: "C", Operator: metadata.ConditionRegEqual, Value: []string{"test.*"}}, + }, + { + {DimensionName: "D", Operator: metadata.ConditionLte, Value: []string{"100"}}, + {DimensionName: "E", Operator: metadata.ConditionContains, Value: []string{"x", "y"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`A` = '1' AND `B` != '2' OR `C` REGEXP 'test.*' OR `D` <= 100 AND `E` IN ('x', 'y'))" + assert.Equal(t, expected, result) + }) + + t.Run("单值IN转换测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "version", Operator: metadata.ConditionEqual, Value: []string{"v1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`version` = 'v1'", result) + }) + + t.Run("正则表达式组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "path", Operator: metadata.ConditionRegEqual, Value: []string{"^/api", "v2$"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`path` REGEXP '^/api|v2$'", result) + }) +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go new file mode 100644 index 000000000..13ad0ced1 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -0,0 +1,552 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" +) + +const ( + Doris = "doris" + DorisTypeText = "text" + + ShardKey = "__shard_key__" + + DefaultKey = "log" +) + +type DorisSQLExpr struct { + encodeFunc func(string) string + + timeField string + valueField string + + keepColumns []string + fieldsMap map[string]string + + isSetLabels bool + lock sync.Mutex + labelCheck map[string]struct{} + labelMap map[string][]string +} + +var _ SQLExpr = (*DorisSQLExpr)(nil) + +func (d *DorisSQLExpr) Type() string { + return Doris +} + +func (d *DorisSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { + d.timeField = timeField + d.valueField = valueField + return d +} + +func (d *DorisSQLExpr) WithEncode(fn func(string) string) SQLExpr { + d.encodeFunc = fn + return d +} + +func (d *DorisSQLExpr) IsSetLabels(isSetLabels bool) SQLExpr { + d.isSetLabels = isSetLabels + return d +} + +func (d *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { + d.fieldsMap = fieldsMap + return d +} + +func (d *DorisSQLExpr) WithKeepColumns(cols []string) SQLExpr { + d.keepColumns = cols + return d +} + +func (d *DorisSQLExpr) FieldMap() map[string]string { + return d.fieldsMap +} + +func (d *DorisSQLExpr) GetLabelMap() map[string][]string { + return d.labelMap +} + +func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { + expr, err := querystring.Parse(qs) + if err != nil { + return "", err + } + if expr == nil { + return "", nil + } + + return d.walk(expr) +} + +func (d *DorisSQLExpr) DescribeTableSQL(table string) string { + return fmt.Sprintf("SHOW CREATE TABLE %s", table) +} + +// ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 +func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, timeAggregate TimeAggregate, err error) { + valueField, _ := d.dimTransform(d.valueField) + + var ( + window time.Duration + offsetMinutes int64 + + timezone string + dimensionMap = map[string]struct{}{ + FieldValue: {}, + FieldTime: {}, + } + ) + + for _, agg := range aggregates { + for _, dim := range agg.Dimensions { + var ( + isObject = false + + newDim string + selectAlias string + ) + newDim, isObject = d.dimTransform(dim) + if isObject && d.encodeFunc != nil { + selectAlias = fmt.Sprintf("%s AS `%s`", newDim, d.encodeFunc(dim)) + newDim = d.encodeFunc(dim) + } else { + selectAlias = newDim + } + + dimensionMap[dim] = struct{}{} + + selectFields = append(selectFields, selectAlias) + groupByFields = append(groupByFields, newDim) + } + + if valueField == "" { + valueField = SelectAll + } + + switch agg.Name { + case "cardinality": + selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + // date_histogram 不支持无需进行函数聚合 + case "date_histogram": + default: + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + } + + if agg.Window > 0 { + window = agg.Window + timezone = agg.TimeZone + } + } + + if window > 0 { + // 获取时区偏移量 + // 如果是按天聚合，则增加时区偏移量 + if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(timezone) + if locErr != nil { + loc = time.UTC + } + _, offset := time.Now().In(loc).Zone() + offsetMinutes = int64(offset) / 60 + } + + // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 + var timeField string + if int64(window.Seconds())%60 == 0 { + windowMinutes := int(window.Minutes()) + timeField = fmt.Sprintf(`((CAST((%s / 1000 + %d) / %d AS INT) * %d - %d) * 60 * 1000)`, ShardKey, offsetMinutes, windowMinutes, windowMinutes, offsetMinutes) + } else { + timeField = fmt.Sprintf(`CAST(%s / %d AS INT) * %d `, d.timeField, window.Milliseconds(), window.Milliseconds()) + } + + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", timeField, TimeStamp)) + groupByFields = append(groupByFields, TimeStamp) + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) + } + + if len(selectFields) == 0 { + if len(d.keepColumns) > 0 { + selectFields = append(selectFields, d.keepColumns...) + } else { + selectFields = append(selectFields, SelectAll) + } + + if valueField != "" { + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, Value)) + } + if d.timeField != "" { + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, TimeStamp)) + } + } + + for _, order := range orders { + // 如果是聚合操作的话，只能使用维度进行排序 + if len(aggregates) > 0 { + if _, ok := dimensionMap[order.Name]; !ok { + continue + } + } + + var orderField string + switch order.Name { + case FieldValue: + orderField = Value + case FieldTime: + orderField = TimeStamp + default: + orderField = order.Name + } + + orderField, _ = d.dimTransform(orderField) + + ascName := "ASC" + if !order.Ast { + ascName = "DESC" + } + orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) + } + + // 回传时间聚合信息 + timeAggregate = TimeAggregate{ + Window: window, + OffsetMillis: offsetMinutes, + } + + return +} + +func (d *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + var ( + orConditions []string + ) + + // 遍历所有OR条件组 + for _, conditions := range allConditions { + var andConditions []string + // 处理每个AND条件组 + for _, cond := range conditions { + buildCondition, err := d.buildCondition(cond) + if err != nil { + return "", err + } + if buildCondition != "" { + andConditions = append(andConditions, buildCondition) + } + } + // 合并AND条件 + if len(andConditions) > 0 { + orConditions = append(orConditions, strings.Join(andConditions, " AND ")) + } + } + + // 处理最终OR条件组合 + if len(orConditions) > 0 { + if len(orConditions) == 1 { + return orConditions[0], nil + } + return fmt.Sprintf("(%s)", strings.Join(orConditions, " OR ")), nil + } + + return "", nil +} + +func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { + if len(c.Value) == 0 { + return "", nil + } + + var ( + oldKey string + key string + op string + val string + ) + + oldKey = c.DimensionName + key, _ = d.dimTransform(oldKey) + + // 对值进行转义处理 + for i, v := range c.Value { + c.Value[i] = d.valueTransform(v) + } + + // 根据操作符类型生成不同的SQL表达式 + switch c.Operator { + // 处理等于类操作符（=, IN, LIKE） + case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) == 1 && c.Value[0] == "" { + op = "IS NULL" + break + } + + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + break + } + + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "LIKE" + } else { + format = "'%s'" + if d.checkMatchALL(c.DimensionName) { + op = "MATCH_PHRASE_PREFIX" + } else { + op = "=" + } + } + + var filter []string + for _, v := range c.Value { + d.addLabel(oldKey, v) + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) + } + // 处理不等于类操作符（!=, NOT IN, NOT LIKE） + case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) == 1 && c.Value[0] == "" { + op = "IS NOT NULL" + break + } + + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { + op = "NOT IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + break + } + + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "NOT LIKE" + } else { + format = "'%s'" + if d.checkMatchALL(c.DimensionName) { + op = "NOT MATCH_PHRASE_PREFIX" + } else { + op = "!=" + } + } + + var filter []string + for _, v := range c.Value { + if v != "" { + d.addLabel(key, v) + } + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) + } + // 处理正则表达式匹配 + case metadata.ConditionRegEqual: + op = "REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) // 多个值用|连接 + case metadata.ConditionNotRegEqual: + op = "NOT REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) + // 处理数值比较操作符（>, >=, <, <=） + case metadata.ConditionGt: + op = ">" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionGte: + op = ">=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLt: + op = "<" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLte: + op = "<=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + default: + return "", fmt.Errorf("unknown operator %s", c.Operator) + } + + if key != "" { + condition := fmt.Sprintf("%s %s", key, op) + if val != "" { + d.addLabel(oldKey, val) + condition = fmt.Sprintf("%s %s", condition, val) + } + return condition, nil + } + return val, nil +} + +func (d *DorisSQLExpr) checkMatchALL(k string) bool { + if d.fieldsMap != nil { + if t, ok := d.fieldsMap[k]; ok { + if t == DorisTypeText { + return true + } + } + } + return false +} + +func (d *DorisSQLExpr) addLabel(key, value string) { + if !d.isSetLabels { + return + } + + d.lock.Lock() + defer d.lock.Unlock() + + if d.labelCheck == nil { + d.labelCheck = make(map[string]struct{}) + } + if d.labelMap == nil { + d.labelMap = make(map[string][]string) + } + + if _, ok := d.labelCheck[key+value]; !ok { + d.labelCheck[key+value] = struct{}{} + d.labelMap[key] = append(d.labelMap[key], value) + } +} + +func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { + var ( + err error + left string + right string + ) + + switch c := e.(type) { + case *querystring.NotExpr: + left, err = d.walk(c.Expr) + if err != nil { + return "", err + } + return fmt.Sprintf("NOT (%s)", left), nil + case *querystring.OrExpr: + left, err = d.walk(c.Left) + if err != nil { + return "", err + } + right, err = d.walk(c.Right) + if err != nil { + return "", err + } + return fmt.Sprintf("(%s OR %s)", left, right), nil + case *querystring.AndExpr: + left, err = d.walk(c.Left) + if err != nil { + return "", err + } + right, err = d.walk(c.Right) + if err != nil { + return "", err + } + return fmt.Sprintf("%s AND %s", left, right), nil + case *querystring.WildcardExpr: + if c.Field == "" { + c.Field = DefaultKey + } + + d.addLabel(c.Field, c.Value) + field, _ := d.dimTransform(c.Field) + return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil + case *querystring.MatchExpr: + if c.Field == "" { + c.Field = DefaultKey + } + d.addLabel(c.Field, c.Value) + field, _ := d.dimTransform(c.Field) + if d.checkMatchALL(c.Field) { + return fmt.Sprintf("%s MATCH_PHRASE_PREFIX '%s'", field, c.Value), nil + } + + return fmt.Sprintf("%s = '%s'", field, c.Value), nil + case *querystring.NumberRangeExpr: + if c.Field == "" { + c.Field = DefaultKey + } + field, _ := d.dimTransform(c.Field) + var timeFilter []string + if c.Start != nil && *c.Start != "*" { + var op string + if c.IncludeStart { + op = ">=" + } else { + op = ">" + } + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", field, op, *c.Start)) + } + + if c.End != nil && *c.End != "*" { + var op string + if c.IncludeEnd { + op = "<=" + } else { + op = "<" + } + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", field, op, *c.End)) + } + + return fmt.Sprintf("%s", strings.Join(timeFilter, " AND ")), nil + default: + err = fmt.Errorf("expr type is not match %T", e) + } + + return "", err +} + +func (d *DorisSQLExpr) dimTransform(s string) (string, bool) { + if s == "" { + return "", false + } + + fs := strings.Split(s, ".") + if len(fs) > 1 { + return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `.`)), true + } + return fmt.Sprintf("`%s`", s), false +} + +func (d *DorisSQLExpr) valueTransform(s string) string { + if strings.Contains(s, "'") { + s = strings.ReplaceAll(s, "'", "''") + } + return s +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go new file mode 100644 index 000000000..c3d3fb9e9 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -0,0 +1,260 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +func TestDorisSQLExpr_ParserQueryString(t *testing.T) { + tests := []struct { + name string + input string + want string + err string + }{ + { + name: "simple match", + input: "name:test", + want: "`name` = 'test'", + }, + { + name: "one word", + input: "test", + want: "`log` = 'test'", + //err: "doris 不支持全字段检索: test", + }, + { + name: "complex nested query", + input: "(a:1 AND (b:2 OR c:3)) OR NOT d:4", + want: "(`a` = '1' AND (`b` = '2' OR `c` = '3') OR NOT (`d` = '4'))", + }, + { + name: "invalid syntax", + input: "name:test AND OR", + err: "syntax error: unexpected tOR", + }, + { + name: "empty input", + input: "", + }, + { + name: "OR expression with multiple terms", + input: "a:1 OR b:2 OR c:3", + want: "(`a` = '1' OR (`b` = '2' OR `c` = '3'))", + }, + { + name: "mixed AND/OR with proper precedence", + input: "a:1 AND b:2 OR c:3", + want: "`a` = '1' AND (`b` = '2' OR `c` = '3')", + }, + { + name: "exact match with quotes", + input: "name:\"exact match\"", + want: "`name` = 'exact match'", + }, + { + name: "numeric equality", + input: "age:25", + want: "`age` = '25'", + }, + { + name: "date range query", + input: "timestamp:[2023-01-01 TO 2023-12-31]", + want: "( timestamp >= '2023-01-01' AND timestamp <= '2023-12-31' )", + err: "syntax error: unexpected tSTRING, expecting tPHRASE or tNUMBER or tSTAR or tMINUS", + }, + { + name: "invalid field name", + input: "123field:value", + want: "`123field` = 'value'", + }, + { + name: "text filter", + input: "text:value", + want: "`text` MATCH_PHRASE_PREFIX 'value'", + }, + { + name: "object field", + input: "__ext.container_name: value", + want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", + }, + { + name: "start", + input: "a: >100", + want: "`a` > 100", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sqlExpr.NewSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "text": sqlExpr.DorisTypeText, + }).ParserQueryString(tt.input) + if err != nil { + assert.Equal(t, err.Error(), tt.err) + return + } + + assert.Equal(t, tt.want, got) + }) + } +} + +// TestDorisSQLExpr_ParserAllConditions 单元测试 +func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { + tests := []struct { + name string + condition metadata.AllConditions + want string + wantErr error + }{ + { + name: "doris test object field condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"What's UP"}, + Operator: metadata.ConditionEqual, + }, + { + DimensionName: "tag", + Value: []string{"test"}, + Operator: metadata.ConditionNotEqual, + }, + }, + }, + want: "CAST(object[\"field\"] AS STRING) MATCH_PHRASE_PREFIX 'What''s UP' AND `tag` != 'test'", + }, + { + name: "doris t8est text field wildcard", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"partial"}, + Operator: metadata.ConditionContains, + IsWildcard: true, + }, + }, + }, + want: "CAST(object[\"field\"] AS STRING) LIKE '%partial%'", + }, + { + name: "doris test OR condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "status", + Value: []string{"running"}, + Operator: metadata.ConditionEqual, + }, + }, + { + { + DimensionName: "code", + Value: []string{"500"}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "(`status` = 'running' OR `code` = '500')", + }, + { + name: "doris test numeric field without cast", + condition: metadata.AllConditions{ + { + { + DimensionName: "cpu_usage", + Value: []string{"80"}, + Operator: metadata.ConditionGt, + }, + }, + }, + want: "`cpu_usage` > 80", + }, + { + name: "test IN operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "env", + Value: []string{"prod", "test"}, + Operator: metadata.ConditionContains, + }, + }, + }, + want: "`env` IN ('prod', 'test')", + }, + { + name: "test IN operator with wildcard", + condition: metadata.AllConditions{ + { + { + DimensionName: "env", + Value: []string{"prod", "test"}, + Operator: metadata.ConditionContains, + IsWildcard: true, + }, + }, + }, + want: "(`env` LIKE '%prod%' OR `env` LIKE '%test%')", + }, + { + name: "doris test empty value", + condition: metadata.AllConditions{ + { + { + DimensionName: "host", + Value: []string{}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "", + }, + { + name: "doris test invalid operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "time", + Value: []string{"2023"}, + Operator: "unknown", + }, + }, + }, + wantErr: fmt.Errorf("unknown operator unknown"), + }, + } + + e := sqlExpr.NewSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "object.field": sqlExpr.DorisTypeText, + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := e.ParserAllConditions(tt.condition) + + if err != nil { + assert.Equal(t, tt.wantErr, err) + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index dc40ba5ce..534a48020 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -183,6 +183,10 @@ func (i *Instance) getMappings(ctx context.Context, conn Connect, aliases []stri return nil, err } mappingMap, err := client.GetMapping().Index(aliases...).Type("").Do(ctx) + if err != nil { + log.Warnf(ctx, "get mapping error: %s", err.Error()) + return nil, err + } indexes := make([]string, 0, len(mappingMap)) for index := range mappingMap { @@ -231,7 +235,7 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac // querystring 生成 elastic.query if qb.QueryString != "" { qs := NewQueryString(qb.QueryString, fact.NestedField) - q, qsErr := qs.Parser() + q, qsErr := qs.ToDSL() if qsErr != nil { return nil, qsErr } @@ -648,7 +652,7 @@ func (i *Instance) QuerySeriesSet( err error ) - ctx, span := trace.NewSpan(ctx, "elasticsearch-query-reference") + ctx, span := trace.NewSpan(ctx, "elasticsearch-query-series-set") defer span.End(&err) if len(query.Aggregates) == 0 { @@ -717,10 +721,10 @@ func (i *Instance) QuerySeriesSet( conn: conn, } - mappings, err1 := i.getMappings(ctx, qo.conn, qo.indexes) + mappings, errMapping := i.getMappings(ctx, qo.conn, qo.indexes) // index 不存在，mappings 获取异常直接返回空 if len(mappings) == 0 { - log.Warnf(ctx, "index is empty with %v", qo.indexes) + log.Warnf(ctx, "index is empty with %v with %s error %v", qo.indexes, qo.conn.String(), errMapping) return } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index 34e101d66..f39b424aa 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -13,9 +13,9 @@ import ( "context" "fmt" - parser "github.com/bytedance/go-querystring-parser" elastic "github.com/olivere/elastic/v7" + qs "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) @@ -46,10 +46,10 @@ func (s *QueryString) queryString(str string) elastic.Query { return elastic.NewQueryStringQuery(str).AnalyzeWildcard(true).Field("*").Field("__*").Lenient(true) } -func (s *QueryString) Parser() (elastic.Query, error) { +func (s *QueryString) ToDSL() (elastic.Query, error) { defer func() { if r := recover(); r != nil { - log.Errorf(context.TODO(), "query string (%v) parser error", s.q) + log.Errorf(context.TODO(), "querystring(%s) todsl panic: %v", s.q, r) } }() @@ -58,21 +58,20 @@ func (s *QueryString) Parser() (elastic.Query, error) { } // 解析失败，或者没有 nested 字段，则使用透传的方式查询 - qs := s.queryString(s.q) - - ast, err := parser.Parse(s.q) + q := s.queryString(s.q) + ast, err := qs.Parse(s.q) if err != nil { - return qs, nil + return q, nil } conditionQuery, err := s.walk(ast) if err != nil { - return qs, nil + return q, nil } // 如果 nestedFields 不存在则直接使用 queryString 透传 if len(s.nestedFields) == 0 { - return qs, nil + return q, nil } for nestedKey := range s.nestedFields { @@ -90,20 +89,20 @@ func (s *QueryString) check(field string) { } } -func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { +func (s *QueryString) walk(expr qs.Expr) (elastic.Query, error) { var ( leftQ elastic.Query rightQ elastic.Query err error ) - switch c := condition.(type) { - case *parser.NotCondition: - leftQ, err = s.walk(c.Condition) + switch c := expr.(type) { + case *qs.NotExpr: + leftQ, err = s.walk(c.Expr) if err != nil { return nil, err } leftQ = elastic.NewBoolQuery().MustNot(leftQ) - case *parser.OrCondition: + case *qs.OrExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -113,7 +112,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Should(leftQ, rightQ) - case *parser.AndCondition: + case *qs.AndExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -123,15 +122,19 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Must(leftQ, rightQ) - case *parser.MatchCondition: + case *qs.MatchExpr: if c.Field != "" { leftQ = elastic.NewMatchPhraseQuery(c.Field, c.Value) s.check(c.Field) } else { leftQ = s.queryString(fmt.Sprintf(`"%s"`, c.Value)) } - case *parser.NumberRangeCondition: + case *qs.NumberRangeExpr: q := elastic.NewRangeQuery(c.Field) + if c.Start == nil && c.End == nil { + return nil, fmt.Errorf("start and end is nil") + } + if c.Start != nil { if c.IncludeStart { q.Gte(*c.Start) @@ -139,7 +142,6 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { q.Gt(*c.Start) } } - if c.End != nil { if c.IncludeEnd { q.Lte(*c.End) @@ -149,7 +151,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { } s.check(c.Field) leftQ = q - case *parser.WildcardCondition: + case *qs.WildcardExpr: if c.Field != "" { leftQ = elastic.NewWildcardQuery(c.Field, c.Value) s.check(c.Field) @@ -157,7 +159,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { leftQ = s.queryString(c.Value) } default: - err = fmt.Errorf("condition type is not match %T", condition) + err = fmt.Errorf("expr type is not match %T", expr) } return leftQ, err } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 1311f3348..9d9b59612 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -59,7 +59,8 @@ func TestQsToDsl(t *testing.T) { expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, }, { - q: `*`, + q: `*`, + expected: `{"query_string":{"fields":["*","__*"],"analyze_wildcard":true,"lenient":true,"query":"*"}}`, }, { q: ``, @@ -77,13 +78,19 @@ func TestQsToDsl(t *testing.T) { } return "" }) - query, err := qs.Parser() + query, err := qs.ToDSL() if err == nil { if query != nil { - body, _ := query.Source() - bodyJson, _ := json.Marshal(body) - bodyString := string(bodyJson) - assert.JSONEq(t, c.expected, bodyString) + body, err := query.Source() + assert.Nil(t, err) + + if body != nil { + bodyJson, _ := json.Marshal(body) + bodyString := string(bodyJson) + assert.JSONEq(t, c.expected, bodyString) + } else { + assert.Empty(t, c.expected) + } } } else { assert.Equal(t, c.err, err) diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go index 61da29e76..b07efb67c 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go @@ -31,7 +31,7 @@ var ( vmRt string = "2_bcs_prom_computation_result_table_00000" instance = &Instance{ - url: mock.VmUrl, + url: mock.BkBaseUrl, timeout: time.Minute * 5, curl: &curl.HttpCurl{}, } @@ -103,7 +103,7 @@ func TestInstance_DirectQueryRange(t *testing.T) { }) instance := &Instance{ - url: mock.VmUrl, + url: mock.BkBaseUrl, timeout: time.Minute * 5, curl: &curl.HttpCurl{}, }