From e2c2f9121b7a20b43a736461cb5746da78029ef9 Mon Sep 17 00:00:00 2001 From: ctenetlau <74183504+EASYGOING45@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:21:55 +0800 Subject: [PATCH 001/104] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=85=B1?= =?UTF-8?q?=E4=BA=AB=E9=9B=86=E7=BE=A4=E8=BF=87=E6=BB=A4=E6=97=A0=E6=95=88?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20--bug=3D129763639=20(#505)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata/service/spaceredis_test.go | 141 ++++++++++++++++++ .../utils/optionx/options.go | 28 +++- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go b/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go index fcbed9e01..634388473 100644 --- a/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go +++ b/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go @@ -58,6 +58,147 @@ func TestSpacePusher_getMeasurementType(t *testing.T) { } } +func TestSpacePusher_composeBcsSpaceClusterTableIds(t *testing.T) { + // 初始化测试数据库配置 + mocker.InitTestDBConfig("../../../bmw_test.yaml") + db := mysql.GetDBSession().DB + + // 创建一个真实的SpaceResource数据 + resourceId := "monitor" + dimensionValues := `[{"cluster_id": "BCS-K8S-00000", "namespace": null, "cluster_type": "single"}, + {"cluster_id": "BCS-K8S-00001", "namespace": ["bkm-test-4"], "cluster_type": "shared"}, + {"cluster_id": "BCS-K8S-00002", "namespace": ["bkm-test-1", "bkm-test-2", "bkm-test-3"], "cluster_type": "shared"}, + {"cluster_id": "BCS-K8S-00003", "namespace": [], "cluster_type": "shared"}]` + spaceResource := space.SpaceResource{ + Id: 207, + SpaceTypeId: "bkci", + SpaceId: "monitor", + ResourceType: "bcs", + ResourceId: &resourceId, + DimensionValues: dimensionValues, + } + db.Delete(&spaceResource) + err := db.Create(&spaceResource).Error + assert.NoError(t, err) + + // 创建 BCSClusterInfo 数据 + clusterInfos := []bcs.BCSClusterInfo{ + { + ClusterID: "BCS-K8S-00000", + K8sMetricDataID: 1001, + CustomMetricDataID: 2001, + }, + { + ClusterID: "BCS-K8S-00001", + K8sMetricDataID: 1002, + CustomMetricDataID: 2002, + }, + { + ClusterID: "BCS-K8S-00002", + K8sMetricDataID: 1003, + CustomMetricDataID: 2003, + }, + { + ClusterID: "BCS-K8S-00003", + K8sMetricDataID: 1004, + CustomMetricDataID: 2004, + }, + } + db.Delete(&bcs.BCSClusterInfo{}) + for _, ci := range clusterInfos { + err = db.Create(&ci).Error + assert.NoError(t, err) + } + + // 创建 DataSourceResultTable 数据 + dataSourceResultTables := []resulttable.DataSourceResultTable{ + { + BkDataId: 1001, + TableId: "table1", + }, + { + BkDataId: 2001, + TableId: "table2", + }, + { + BkDataId: 1002, + TableId: "table3", + }, + { + BkDataId: 2002, + TableId: "table4", + }, + { + BkDataId: 1003, + TableId: "table5", + }, + { + BkDataId: 2003, + TableId: "table6", + }, + { + BkDataId: 1004, + TableId: "table7", + }, + { + BkDataId: 2004, + TableId: "table8", + }, + } + db.Delete(&resulttable.DataSourceResultTable{}) + for _, dsrt := range dataSourceResultTables { + err = db.Create(&dsrt).Error + assert.NoError(t, err) + } + + // 执行被测试的方法 + spacePusher := NewSpacePusher() + result, err := spacePusher.composeBcsSpaceClusterTableIds("bkci", "monitor") + assert.NoError(t, err) + + // 输出调试信息 + fmt.Printf("Result: %+v\n", result) + + expectedResults := map[string]map[string]interface{}{ + "table1": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00000", "namespace": nil}, + }, + }, + "table2": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00000", "namespace": nil}, + }, + }, + "table3": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00001", "namespace": "bkm-test-4"}, + }, + }, + "table4": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00001", "namespace": "bkm-test-4"}, + }, + }, + "table5": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-1"}, + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-2"}, + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-3"}, + }, + }, + "table6": { + "filters": []map[string]interface{}{ + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-1"}, + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-2"}, + {"bcs_cluster_id": "BCS-K8S-00002", "namespace": "bkm-test-3"}, + }, + }, + } + + assert.Equal(t, expectedResults, result) +} + func TestSpacePusher_refineTableIds(t *testing.T) { mocker.InitTestDBConfig("../../../bmw_test.yaml") db := mysql.GetDBSession().DB diff --git a/pkg/bk-monitor-worker/utils/optionx/options.go b/pkg/bk-monitor-worker/utils/optionx/options.go index 6acc9b1c3..27a37b590 100644 --- a/pkg/bk-monitor-worker/utils/optionx/options.go +++ b/pkg/bk-monitor-worker/utils/optionx/options.go @@ -11,6 +11,8 @@ package optionx import ( "time" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" ) // Options 用于传递动态参数 @@ -87,9 +89,31 @@ func (o *Options) GetDuration(key string) (time.Duration, bool) { return value, ok } +// GetStringSlice retrieves a string slice from the options func (o *Options) GetStringSlice(key string) ([]string, bool) { - value, ok := o.params[key].([]string) - return value, ok + value, ok := o.params[key] + if !ok { + return nil, false + } + + switch v := value.(type) { + case []string: + return v, true + case []interface{}: + strSlice := make([]string, len(v)) + for i, item := range v { + str, ok := item.(string) + if !ok { + logger.Errorf("Invalid type for key %s, slice failed", key) + return nil, false + } + strSlice[i] = str + } + return strSlice, true + default: + logger.Errorf("Invalid type for key %s, slice failed", key) + return nil, false + } } func (o *Options) GetInterfaceSliceWithString(key string) ([]string, bool) { From 2cc6884f673576767449592676313571e5da07d4 Mon Sep 17 00:00:00 2001 From: ctenetlau <74183504+EASYGOING45@users.noreply.github.com> Date: Wed, 28 Aug 2024 14:51:38 +0800 Subject: [PATCH 002/104] =?UTF-8?q?feat:=20BCS=E5=85=B1=E4=BA=AB=E9=9B=86?= =?UTF-8?q?=E7=BE=A4-=E6=B7=BB=E5=8A=A0=E6=9B=B4=E5=A4=9A=E6=97=A5?= =?UTF-8?q?=E5=BF=97=20--story=3D119412655=20(#507)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/metadata/service/spaceredis_test.go | 7 ++++--- pkg/bk-monitor-worker/utils/optionx/options.go | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go b/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go index 634388473..bd9043cd7 100644 --- a/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go +++ b/pkg/bk-monitor-worker/internal/metadata/service/spaceredis_test.go @@ -71,7 +71,7 @@ func TestSpacePusher_composeBcsSpaceClusterTableIds(t *testing.T) { {"cluster_id": "BCS-K8S-00003", "namespace": [], "cluster_type": "shared"}]` spaceResource := space.SpaceResource{ Id: 207, - SpaceTypeId: "bkci", + SpaceTypeId: models.SpaceTypeBKCI, SpaceId: "monitor", ResourceType: "bcs", ResourceId: &resourceId, @@ -760,11 +760,12 @@ func TestComposeEsTableIdOptions(t *testing.T) { assert.NoError(t, rtOp3.Create(db)) // 获取正常数据 - data := SpacePusher{}.composeEsTableIdOptions([]string{rt1, rt2, rt3}) + spacePusher := NewSpacePusher() + data := spacePusher.composeEsTableIdOptions([]string{rt1, rt2, rt3}) assert.Equal(t, 3, len(data)) assert.Equal(t, map[string]interface{}{"name": "v1"}, data[rt1][rtOp1.Name]) // 获取不存在的rt数据 - data = SpacePusher{}.composeEsTableIdOptions([]string{"not_exist"}) + data = spacePusher.composeEsTableIdOptions([]string{"not_exist"}) assert.Equal(t, 0, len(data)) } diff --git a/pkg/bk-monitor-worker/utils/optionx/options.go b/pkg/bk-monitor-worker/utils/optionx/options.go index 27a37b590..d1e6e70ec 100644 --- a/pkg/bk-monitor-worker/utils/optionx/options.go +++ b/pkg/bk-monitor-worker/utils/optionx/options.go @@ -95,6 +95,10 @@ func (o *Options) GetStringSlice(key string) ([]string, bool) { if !ok { return nil, false } + // 针对Single集群，namespace存在为None的场景 + if value == nil { + return nil, true + } switch v := value.(type) { case []string: @@ -104,14 +108,14 @@ func (o *Options) GetStringSlice(key string) ([]string, bool) { for i, item := range v { str, ok := item.(string) if !ok { - logger.Errorf("Invalid type for key %s, slice failed", key) + logger.Errorf("Invalid type for key %s, got %T; overall value: %#v", key, item, v) return nil, false } strSlice[i] = str } return strSlice, true default: - logger.Errorf("Invalid type for key %s, slice failed", key) + logger.Errorf("Invalid type for key %s, got %T", key, value) return nil, false } } From c2a64e3624bbc0069960b1986a958ad3e20166a0 Mon Sep 17 00:00:00 2001 From: dongdong Date: Thu, 29 Aug 2024 10:08:15 +0800 Subject: [PATCH 003/104] sync: master branch (#509) Co-authored-by: LDepression <1197285120@qq.com> Co-authored-by: frostycliu Co-authored-by: Shamcle Ren Co-authored-by: ctenetlau <74183504+EASYGOING45@users.noreply.github.com> --- .../internal/alarm/cmdbcache/module.go | 3 +- pkg/bk-monitor-worker/script/diff/go.mod | 2 +- pkg/bk-monitor-worker/script/diff/go.sum | 1 + pkg/bkmonitorbeat/VERSION | 2 +- pkg/bkmonitorbeat/configs/httpconfig.go | 11 + pkg/bkmonitorbeat/configs/simpletaskconfig.go | 11 + pkg/bkmonitorbeat/tasks/http/event.go | 89 ++ pkg/bkmonitorbeat/tasks/http/gather.go | 260 ++--- pkg/bkmonitorbeat/tasks/http/gather_test.go | 7 +- .../module/prometheus/collector/action.go | 131 +++ .../prometheus/collector/action_test.go | 125 +++ .../module/prometheus/collector/collector.go | 206 ++-- .../prometheus/collector/collector_test.go | 9 - .../module/prometheus/collector/relabel.go | 63 +- pkg/bkmonitorbeat/tasks/prom_event.go | 72 +- pkg/bkmonitorbeat/tasks/script/formatdata.go | 4 +- .../tasks/script/formatdata_test.go | 8 +- pkg/bkmonitorbeat/tasks/script/gather.go | 6 +- pkg/bkmonitorbeat/tasks/tcp/gather.go | 21 +- pkg/bkmonitorbeat/tasks/udp/gather.go | 18 +- pkg/operator/README.md | 2 + pkg/operator/common/eplabels/labels.go | 38 + pkg/operator/docs/API.md | 26 +- pkg/operator/docs/check-help.md | 442 ++++++++ pkg/operator/docs/dataId.md | 85 ++ pkg/operator/docs/feature.md | 241 +++++ pkg/operator/docs/metrics.md | 86 +- pkg/operator/go.mod | 1 + pkg/operator/go.sum | 1 + pkg/operator/operator/discover/discover.go | 10 +- pkg/operator/operator/discover/endpoint.go | 24 +- pkg/operator/operator/discover/hook.go | 4 +- pkg/operator/operator/hook.go | 7 +- pkg/operator/operator/metrics.go | 13 + .../operator/objectsref/controller.go | 8 +- pkg/operator/operator/objectsref/extend.go | 2 - pkg/operator/operator/objectsref/metrics.go | 18 - pkg/operator/operator/operator.go | 33 +- pkg/operator/operator/promsli/promsli.go | 46 +- pkg/operator/operator/relabel.go | 8 +- pkg/operator/operator/server.go | 27 +- pkg/operator/operator/statefulset.go | 2 +- pkg/operator/operator/target/event_test.go | 1 + pkg/unify-query/bkapi/bkapi.go | 65 ++ pkg/unify-query/bkapi/bkdata.go | 90 ++ pkg/unify-query/bkapi/settings.go | 22 + pkg/unify-query/cmdb/v1beta1/v1beta1.go | 4 +- pkg/unify-query/cmdb/v1beta1/v1beta1_test.go | 10 +- pkg/unify-query/docs/docs.go | 2 +- pkg/unify-query/metadata/reference_test.go | 2 +- pkg/unify-query/metadata/struct.go | 1 - pkg/unify-query/query/interfaces.go | 4 + pkg/unify-query/query/structured/query_ts.go | 9 +- pkg/unify-query/query/structured/space.go | 1 + pkg/unify-query/service/http/check_handler.go | 2 +- pkg/unify-query/service/http/promql.go | 1 + pkg/unify-query/service/http/query.go | 10 +- pkg/unify-query/service/http/query_test.go | 4 +- pkg/unify-query/service/http/tsquery_test.go | 2 +- pkg/unify-query/service/tsdb/service.go | 70 -- pkg/unify-query/tsdb/bksql/client.go | 108 +- pkg/unify-query/tsdb/bksql/client_test.go | 107 +- pkg/unify-query/tsdb/bksql/instance.go | 160 +-- pkg/unify-query/tsdb/bksql/instance_test.go | 4 +- pkg/unify-query/tsdb/bksql/struct.go | 6 +- .../tsdb/elasticsearch/agg_format.go | 10 +- pkg/unify-query/tsdb/elasticsearch/format.go | 24 +- .../tsdb/elasticsearch/instance.go | 29 +- .../tsdb/elasticsearch/instance_test.go | 30 +- .../tsdb/elasticsearch/query_string.go | 4 +- .../tsdb/elasticsearch/query_string_test.go | 12 +- pkg/unify-query/tsdb/influxdb/instance.go | 9 +- .../tsdb/influxdb/instance_test.go | 6 +- .../tsdb/prometheus/instance_test.go | 951 ++++++++++++++++++ pkg/unify-query/tsdb/prometheus/querier.go | 164 +-- pkg/unify-query/tsdb/storage.go | 3 - .../tsdb/victoriaMetrics/instance.go | 136 +-- .../tsdb/victoriaMetrics/instance_test.go | 80 +- .../tsdb/victoriaMetrics/struct.go | 8 - pkg/unify-query/unify-query.yaml | 8 + pkg/utils/router/influxdb/space.go | 1 + pkg/utils/router/influxdb/space_gen.go | 35 +- 82 files changed, 3240 insertions(+), 1128 deletions(-) create mode 100644 pkg/bkmonitorbeat/tasks/http/event.go create mode 100644 pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action.go create mode 100644 pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action_test.go create mode 100644 pkg/operator/common/eplabels/labels.go create mode 100644 pkg/operator/docs/check-help.md create mode 100644 pkg/operator/docs/dataId.md create mode 100644 pkg/operator/docs/feature.md create mode 100644 pkg/unify-query/bkapi/bkapi.go create mode 100644 pkg/unify-query/bkapi/bkdata.go create mode 100644 pkg/unify-query/bkapi/settings.go diff --git a/pkg/bk-monitor-worker/internal/alarm/cmdbcache/module.go b/pkg/bk-monitor-worker/internal/alarm/cmdbcache/module.go index 7491f5ce6..9c097b7da 100644 --- a/pkg/bk-monitor-worker/internal/alarm/cmdbcache/module.go +++ b/pkg/bk-monitor-worker/internal/alarm/cmdbcache/module.go @@ -34,11 +34,10 @@ import ( "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bk-monitor-worker/internal/alarm/redis" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bk-monitor-worker/internal/api" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bk-monitor-worker/internal/api/cmdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" ) const ( diff --git a/pkg/bk-monitor-worker/script/diff/go.mod b/pkg/bk-monitor-worker/script/diff/go.mod index 44dc40be8..90f01ef73 100644 --- a/pkg/bk-monitor-worker/script/diff/go.mod +++ b/pkg/bk-monitor-worker/script/diff/go.mod @@ -16,7 +16,7 @@ require ( require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic v1.12.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/pkg/bk-monitor-worker/script/diff/go.sum b/pkg/bk-monitor-worker/script/diff/go.sum index d8e6d20fd..319db9e67 100644 --- a/pkg/bk-monitor-worker/script/diff/go.sum +++ b/pkg/bk-monitor-worker/script/diff/go.sum @@ -25,6 +25,7 @@ github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic v1.12.1/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= diff --git a/pkg/bkmonitorbeat/VERSION b/pkg/bkmonitorbeat/VERSION index 1b4940759..a7037038a 100644 --- a/pkg/bkmonitorbeat/VERSION +++ b/pkg/bkmonitorbeat/VERSION @@ -1 +1 @@ -v3.46.x \ No newline at end of file +v3.47.x \ No newline at end of file diff --git a/pkg/bkmonitorbeat/configs/httpconfig.go b/pkg/bkmonitorbeat/configs/httpconfig.go index 925e2f264..ce6fb3287 100644 --- a/pkg/bkmonitorbeat/configs/httpconfig.go +++ b/pkg/bkmonitorbeat/configs/httpconfig.go @@ -34,6 +34,17 @@ type HTTPTaskStepConfig struct { ResponseCodeList []int `config:"response_code_list"` } +func (c *HTTPTaskStepConfig) URLs() []string { + if c == nil { + return nil + } + + if len(c.URLList) > 0 { + return c.URLList + } + return []string{c.URL} +} + // Clean : func (c *HTTPTaskStepConfig) Clean() error { err := c.SimpleMatchParam.CleanParams() diff --git a/pkg/bkmonitorbeat/configs/simpletaskconfig.go b/pkg/bkmonitorbeat/configs/simpletaskconfig.go index f54c8556d..23ce3fb9c 100644 --- a/pkg/bkmonitorbeat/configs/simpletaskconfig.go +++ b/pkg/bkmonitorbeat/configs/simpletaskconfig.go @@ -91,6 +91,17 @@ type SimpleTaskParam struct { TargetPort int `config:"target_port" validate:"required,min=1"` } +func (c *SimpleTaskParam) Hosts() []string { + if c == nil { + return nil + } + + if len(c.TargetHostList) > 0 { + return c.TargetHostList + } + return []string{c.TargetHost} +} + // CleanParams : func (c *SimpleTaskParam) CleanParams() error { if len(c.TargetHostList) > 0 { diff --git a/pkg/bkmonitorbeat/tasks/http/event.go b/pkg/bkmonitorbeat/tasks/http/event.go new file mode 100644 index 000000000..c4b671ce8 --- /dev/null +++ b/pkg/bkmonitorbeat/tasks/http/event.go @@ -0,0 +1,89 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package http + +import ( + "net/url" + "time" + + "github.com/elastic/beats/libbeat/common" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bkmonitorbeat/configs" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bkmonitorbeat/define" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bkmonitorbeat/tasks" +) + +type Event struct { + *tasks.Event + URL string + Index int + Steps int + Method string + ResponseCode int + Message string + Charset string + ContentLength int + MediaType string + ResolvedIP string +} + +func NewEvent(g *Gather) *Event { + conf := g.GetConfig().(*configs.HTTPTaskConfig) + evt := tasks.NewEvent(g) + evt.StartAt = time.Now() + + event := &Event{ + Event: evt, + Steps: len(conf.Steps), + Index: 1, + } + return event +} + +func (e *Event) AsMapStr() common.MapStr { + mapStr := e.Event.AsMapStr() + mapStr["url"] = e.URL + mapStr["steps"] = e.Steps + mapStr["method"] = e.Method + mapStr["response_code"] = e.ResponseCode + mapStr["message"] = e.Message + mapStr["charset"] = e.Charset + mapStr["content_length"] = e.ContentLength + mapStr["media_type"] = e.MediaType + mapStr["resolved_ip"] = e.ResolvedIP + return mapStr +} + +func (e *Event) ToStep(index int, method, url string) { + e.URL = url + e.Method = method + e.Index = index +} + +func (e *Event) OK() bool { + return e.Status == define.GatherStatusOK +} + +func (e *Event) Fail(code define.NamedCode) { + e.Event.Fail(code) + e.Status = int32(e.Index) +} + +func (e *Event) FailFromError(err error) { + e.Message = err.Error() + switch typ := err.(type) { + case *url.Error: + if typ.Timeout() { + e.Fail(define.CodeRequestTimeout) + } else { + e.Fail(define.CodeResponseFailed) + } + } +} diff --git a/pkg/bkmonitorbeat/tasks/http/gather.go b/pkg/bkmonitorbeat/tasks/http/gather.go index 1ef9ce942..8f99660a2 100644 --- a/pkg/bkmonitorbeat/tasks/http/gather.go +++ b/pkg/bkmonitorbeat/tasks/http/gather.go @@ -26,7 +26,6 @@ import ( "sync" "time" - "github.com/elastic/beats/libbeat/common" "github.com/pkg/errors" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bkmonitorbeat/configs" @@ -36,74 +35,35 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" ) -// Event 结果数据 -type Event struct { - *tasks.Event - URL string - Index int - Steps int - Method string - ResponseCode int - Message string - Charset string - ContentLength int - MediaType string - ResolvedIP string -} - -func (e *Event) AsMapStr() common.MapStr { - mapStr := e.Event.AsMapStr() - mapStr["url"] = e.URL - mapStr["steps"] = e.Steps - mapStr["method"] = e.Method - mapStr["response_code"] = e.ResponseCode - mapStr["message"] = e.Message - mapStr["charset"] = e.Charset - mapStr["content_length"] = e.ContentLength - mapStr["media_type"] = e.MediaType - mapStr["resolved_ip"] = e.ResolvedIP - return mapStr -} - -// ToStep 按照采集子配置填写事件信息 -func (e *Event) ToStep(index int, step *configs.HTTPTaskStepConfig, url string) { - e.URL = url - e.Method = step.Method - e.Index = index -} - -func (e *Event) OK() bool { - return e.Status == define.GatherStatusOK -} - -func (e *Event) Fail(code define.NamedCode) { - e.Event.Fail(code) - e.Status = int32(e.Index) -} +// validateConfig 校验配置 解析 base64 内容 +func validateConfig(c *configs.HTTPTaskStepConfig) { + const base64Prefix = "base64://" -func (e *Event) FailFromError(err error) { - e.Message = err.Error() - switch typ := err.(type) { - case *url.Error: - if typ.Timeout() { - e.Fail(define.CodeRequestTimeout) - } else { - e.Fail(define.CodeResponseFailed) + decode := func(s string) string { + s = s[len(base64Prefix):] + b, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(s, "=")) + if err != nil { + return s // 解析失败原路返回 } + return string(b) } -} -func NewEvent(g *Gather) *Event { - conf := g.GetConfig().(*configs.HTTPTaskConfig) - evt := tasks.NewEvent(g) - evt.StartAt = time.Now() + if strings.HasPrefix(c.Request, base64Prefix) { + c.Request = decode(c.Request) + } + if strings.HasPrefix(c.Response, base64Prefix) { + c.Response = decode(c.Response) + } - event := &Event{ - Event: evt, - Steps: len(conf.Steps), - Index: 1, + headers := make(map[string]string) + for k, v := range c.Headers { + if strings.HasPrefix(v, base64Prefix) { + headers[k] = decode(v) + } else { + headers[k] = v + } } - return event + c.Headers = headers } // makeResponseReader 从 response 获取 reader @@ -124,6 +84,19 @@ func makeResponseReader(response *http.Response) io.ReadCloser { return responseRd } +// checkResponseCode 检查返回是否符合配置 +func checkResponseCode(step *configs.HTTPTaskStepConfig, response *http.Response) bool { + if len(step.ResponseCodeList) > 0 { + for _, code := range step.ResponseCodeList { + if response.StatusCode == code { + return true + } + } + return false + } + return true +} + type Gather struct { tasks.BaseTask contentTypeRegexp *regexp.Regexp @@ -149,36 +122,6 @@ func (g *Gather) UpdateEventByResponse(event *Event, response *http.Response) { } } -func validateConfig(c *configs.HTTPTaskStepConfig) { - const base64Prefix = "base64://" - - decode := func(s string) string { - s = s[len(base64Prefix):] - b, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(s, "=")) - if err != nil { - return s // 解析失败原路返回 - } - return string(b) - } - - if strings.HasPrefix(c.Request, base64Prefix) { - c.Request = decode(c.Request) - } - if strings.HasPrefix(c.Response, base64Prefix) { - c.Response = decode(c.Response) - } - - headers := make(map[string]string) - for k, v := range c.Headers { - if strings.HasPrefix(v, base64Prefix) { - headers[k] = decode(v) - } else { - headers[k] = v - } - } - c.Headers = headers -} - // makeRequest 从配置生成请求 func (g *Gather) makeRequest(ctx context.Context, step *configs.HTTPTaskStepConfig, url string) (*http.Request, error) { conf := g.GetConfig().(*configs.HTTPTaskConfig) @@ -202,19 +145,6 @@ func (g *Gather) makeRequest(ctx context.Context, step *configs.HTTPTaskStepConf return request, nil } -// checkResponseCode 检查返回是否符合配置 -func (g *Gather) checkResponseCode(step *configs.HTTPTaskStepConfig, response *http.Response) bool { - if len(step.ResponseCodeList) > 0 { - for _, code := range step.ResponseCodeList { - if response.StatusCode == code { - return true - } - } - return false - } - return true -} - // Client 请求客户端 type Client interface { Do(*http.Request) (*http.Response, error) @@ -244,46 +174,40 @@ func (g *Gather) GatherURL(ctx context.Context, event *Event, step *configs.HTTP // 获取结果 response, err := client.Do(request) if err != nil { - logger.Errorf("request failed, taskid=%v, url=%v, err: %v", conf.TaskID, url, err) + logger.Errorf("task(%d) request failed, url=%v, err: %v", conf.TaskID, url, err) event.FailFromError(err) return false } defer response.Body.Close() - logger.Infof("%v: %v %v response: code=%v", conf.TaskID, step.Method, url, response.StatusCode) - // 根据结果设置事件字段 - g.UpdateEventByResponse(event, response) + logger.Infof("task(%d): %v %v response: code=%v", conf.TaskID, step.Method, url, response.StatusCode) + g.UpdateEventByResponse(event, response) // 根据结果设置事件字段 // 检查响应状态码是否符合预期 - if !g.checkResponseCode(step, response) { + if !checkResponseCode(step, response) { event.Fail(define.CodeResponseNotMatch) return false } // 未配置响应内容无需检查 if step.Response == "" { - logger.Debugf("%v: %v return without match", conf.TaskID, url) event.SuccessOrTimeout() return true } + // 读取响应内容明文reader responseRd := makeResponseReader(response) if responseRd == nil { event.Fail(define.CodeResponseFailed) return false } - defer func() { - err := responseRd.Close() - if err != nil { - logger.Warnf("%v: close response reader error: %v", conf.TaskID, err) - } - }() + defer responseRd.Close() if step.Response != "" { // 读取响应内容字符串 body := g.bufferBuilder.GetBuffer(conf.BufferSize) count, err = responseRd.Read(body) if err != nil && err != io.EOF { - logger.Debugf("%v: %v read response error: %v", conf.TaskID, url, err) + logger.Debugf("task(%d): %v read response error: %v", conf.TaskID, url, err) event.FailFromError(err) return false } @@ -293,15 +217,14 @@ func (g *Gather) GatherURL(ctx context.Context, event *Event, step *configs.HTTP if decoder != nil { decoded, err := decoder.Bytes(body) if err != nil { - logger.Debugf("%v: %v decode body error: %v", conf.TaskID, url, err) + logger.Debugf("task(%d): %v decode body error: %v", conf.TaskID, url, err) body = decoded } } // 对比响应内容是否符合配置 - logger.Debugf("%v: %v response: %s", conf.TaskID, url, body) + logger.Debugf("task(%d): %v response: %s", conf.TaskID, url, body) ok = utils.IsMatch(step.ResponseFormat, body, []byte(step.Response)) if !ok { - logger.Debugf("%v: %v match body fail with type[%v]", conf.TaskID, url, step.ResponseFormat) event.Fail(define.CodeResponseNotMatch) return false } @@ -365,10 +288,8 @@ var NewClient = func(conf *configs.HTTPTaskConfig, proxyMap map[string]string) C return client } -// Run 主入口 func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { conf := g.GetConfig().(*configs.HTTPTaskConfig) - for _, c := range conf.Steps { validateConfig(c) logger.Debugf("validated step config: %#v", c) @@ -377,75 +298,64 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { g.PreRun(ctx) defer g.PostRun(ctx) - pResultMap := make(map[string][]string) for index, step := range conf.Steps { - // 1. 获取 URLList 如果 url 和 urlList 均为空，则直接返回空结果，并且错误码为 success - // 2. 遍历 URLList 逐个 url 判断是 ip 还是域名 - // 1) 是 ip 则直接测试连接获取测试结果 - // 2) 是域名则解析域名，获取 ip 列表： - // dns_check_mode: all - // 1) target_ip_type:0 所有 ip 都测试 - // 2) target_ip_type:4 只测试 ipv4 的ip 若无，则返回错误码 3011 - // 3) target_ip_type:6 只测试 ipv6 的ip 若无，则返回错误码 3012 - // dns_check_mode: single - // 1) target_ip_type:0 取 ip 列表第一个 ip 做测试 - // 2) target_ip_type:4 从 ip 列表中查找是否存在 ipv4 的 ip，存在则取第一个测试，不存在则返回错误码 3011 - // 3) target_ip_type:6 从 ip 列表中查找是否存在 ipv6 的 ip，存在则取第一个测试，不存在则返回错误码 3012 - if step.URL == "" && (len(step.URLList) == 0) { - // 不上报任何数据 - logger.Debugf("http URLList is empty.") - return + urls := step.URLs() + if len(urls) == 0 { + continue } - // 获取配置的url列表 - urls := make([]string, 0) - if step.URL != "" { - urls = append(urls, step.URL) - } - if len(step.URLList) > 0 { - urls = step.URLList - } + // dns_check_mode + // - all: 检查域名解析出来的所有 ip + // - single: 检查域名解析出来的随机一个 ip + resolvedIPs := make(map[string][]string) hostsInfo := tasks.GetHostsInfo(ctx, urls, conf.DNSCheckMode, conf.TargetIPType, configs.Http) for _, h := range hostsInfo { if h.Errno != define.CodeOK { event := NewEvent(g) - event.ToStep(1, step, h.Host) + event.ToStep(index, step.Method, h.Host) event.Fail(h.Errno) e <- event } else { - pResultMap[h.Host] = h.Ips + resolvedIPs[h.Host] = h.Ips } } - // 获取子配置代理配置 + + type Arg struct { + index int + stepConfig *configs.HTTPTaskStepConfig + url string + resolvedIP string + } + + doRequest := func(arg Arg) { + event := NewEvent(g) + event.ToStep(arg.index+1, arg.stepConfig.Method, arg.url) + event.ResolvedIP = arg.resolvedIP + subCtx, cancelFunc := context.WithTimeout(ctx, conf.GetTimeout()) + defer func() { + cancelFunc() + event.EndAt = time.Now() + g.GetSemaphore().Release(1) + e <- event + }() + g.GatherURL(subCtx, event, arg.stepConfig, arg.url, arg.resolvedIP) + } + var wg sync.WaitGroup - for u, ips := range pResultMap { - // 获取并发限制信号量 + for host, ips := range resolvedIPs { err := g.GetSemaphore().Acquire(ctx, int64(len(ips))) if err != nil { - logger.Errorf("semaphore acquire failed for task http task id: %d", g.TaskConfig.GetTaskID()) + logger.Errorf("task(%d) semaphore acquire failed", g.TaskConfig.GetTaskID()) return } - // 按照代理IP列表逐个请求 - for _, ipStr := range ips { + + for _, ip := range ips { wg.Add(1) - go func(i int, s *configs.HTTPTaskStepConfig, url, h string) { - event := NewEvent(g) - event.ToStep(i+1, step, url) - event.ResolvedIP = h - gCtx, cancelFunc := context.WithTimeout(ctx, conf.GetTimeout()) - defer func() { - cancelFunc() - // 设置事件完成时间 - event.EndAt = time.Now() - wg.Done() - // 释放信号量 - g.GetSemaphore().Release(1) - // 发送事件 - e <- event - }() - // 检查 url 并设置结果事件 - g.GatherURL(gCtx, event, s, u, h) - }(index, step, u, ipStr) + arg := Arg{index: index, stepConfig: step, url: host, resolvedIP: ip} + go func() { + defer wg.Done() + doRequest(arg) + }() } } wg.Wait() diff --git a/pkg/bkmonitorbeat/tasks/http/gather_test.go b/pkg/bkmonitorbeat/tasks/http/gather_test.go index 4da2ee07b..428154803 100644 --- a/pkg/bkmonitorbeat/tasks/http/gather_test.go +++ b/pkg/bkmonitorbeat/tasks/http/gather_test.go @@ -135,7 +135,7 @@ func (s *GatherSuite) TestGatherRun() { event := ev.AsMapStr() s.Equal(c.responseCode, event["response_code"]) - s.Equal(c.errorCode, event["error_code"]) + s.Equal(c.errorCode.Code(), event["error_code"]) } } @@ -239,9 +239,8 @@ func (s *GatherSuite) TestUpdateEventByResponse() { for _, tt := range tests { s.T().Run(tt.name, func(t *testing.T) { g := New(tt.fields.globalConfig, tt.fields.taskConfig).(*Gather) - if err := g.UpdateEventByResponse(tt.args.event, tt.args.response); err != nil { - t.Errorf("UpdateEventByResponse() error = %v", err) - } + g.UpdateEventByResponse(tt.args.event, tt.args.response) + s.Equalf(tt.want.mediaType, tt.args.event.MediaType, "UpdateEventByResponse() MediaType = %v, want %v", tt.args.event.MediaType, tt.want.mediaType) s.Equalf(tt.want.charset, tt.args.event.Charset, "UpdateEventByResponse() Charset = %v, want %v", tt.args.event.Charset, tt.want.charset) }) diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action.go new file mode 100644 index 000000000..2da1423b6 --- /dev/null +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action.go @@ -0,0 +1,131 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package collector + +import "sync" + +const ( + ActionTypeDelta = "delta" + ActionTypeRate = "rate" +) + +type ActionConfigs struct { + Rate []ActionRate + Delta ActionDelta +} + +// ActionRate 扩展的 rate action +// 支持对指标计算 rate 进行并生成新指标 +type ActionRate struct { + Source string // 原指标 + Destination string // 新指标 +} + +// ActionDelta 扩展的 delta action +// 支持对指标进行 delta 计算 并且覆盖原指标 value +type ActionDelta []string + +type actionOperator struct { + action string + mut sync.Mutex + + rateKeys map[string]string + deltaKeys map[string]struct{} + + values map[string]map[string]float64 + timestamps map[string]map[string]int64 +} + +func newActionOperator(action string, rateOps []ActionRate, deltaOps ActionDelta) *actionOperator { + rateKeys := make(map[string]string) + for _, op := range rateOps { + rateKeys[op.Source] = op.Destination + } + deltaKeys := make(map[string]struct{}) + for _, op := range deltaOps { + deltaKeys[op] = struct{}{} + } + + return &actionOperator{ + action: action, + rateKeys: rateKeys, + deltaKeys: deltaKeys, + values: make(map[string]map[string]float64), + timestamps: make(map[string]map[string]int64), + } +} + +// GetOrUpdate 获取或者更新数值 +// 如果 bool 为 true 则表示可以直接使用该 value 结果 +func (ao *actionOperator) GetOrUpdate(metric, h string, ts int64, value float64) (string, float64, bool) { + if ao.action == ActionTypeDelta { + newV, ok := ao.getOrUpdateDelta(metric, h, value) + return metric, newV, ok + } + + // ActionTypeRate(default) + return ao.getOrUpdateRate(metric, h, ts, value) +} + +func (ao *actionOperator) getOrUpdateDelta(metric, h string, value float64) (float64, bool) { + ao.mut.Lock() + defer ao.mut.Unlock() + + // 没有命中规则 + if _, ok := ao.deltaKeys[metric]; !ok { + return value, true + } + + if _, ok := ao.values[metric]; !ok { + ao.values[metric] = make(map[string]float64) + } + + v, ok := ao.values[metric][h] + if ok { + deltaV := value - v + ao.values[metric][h] = value + return deltaV, true + } + + ao.values[metric][h] = value + return 0, false +} + +func (ao *actionOperator) getOrUpdateRate(metric, h string, ts int64, value float64) (string, float64, bool) { + ao.mut.Lock() + defer ao.mut.Unlock() + + // 没有命中规则 + newMetric, ok := ao.rateKeys[metric] + if !ok { + return metric, value, true + } + + // values/timestamp 一定同时更新 + if _, ok := ao.values[metric]; !ok { + ao.values[metric] = make(map[string]float64) + ao.timestamps[metric] = make(map[string]int64) + } + + v, ok := ao.values[metric][h] + if ok { + deltaV := value - v + deltaT := ts - ao.timestamps[metric][h] + r := (deltaV) / float64(deltaT) + + ao.values[metric][h] = value + ao.timestamps[metric][h] = ts + return newMetric, r, true + } + + ao.values[metric][h] = value + ao.timestamps[metric][h] = ts + return newMetric, 0, false +} diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action_test.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action_test.go new file mode 100644 index 000000000..d5293f9a1 --- /dev/null +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/action_test.go @@ -0,0 +1,125 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package collector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestActionDelta(t *testing.T) { + type Input struct { + Metric string + Hash string + Ts int64 + Value float64 + } + type Output struct { + Metric string + Value float64 + Ok bool + } + + cases := []struct { + Input Input + Output Output + }{ + { + Input: Input{Metric: "metric3", Hash: "1001", Ts: 1000, Value: 10}, + Output: Output{Metric: "metric3", Value: 10, Ok: true}, + }, + { + Input: Input{Metric: "metric4", Hash: "1001", Ts: 1000, Value: 11}, + Output: Output{Metric: "metric4", Value: 11, Ok: true}, + }, + { + Input: Input{Metric: "metric1", Hash: "1001", Ts: 1000, Value: 11}, + Output: Output{Metric: "metric1", Value: 0, Ok: false}, + }, + { + Input: Input{Metric: "metric1", Hash: "1001", Ts: 1000, Value: 22}, + Output: Output{Metric: "metric1", Value: 11, Ok: true}, + }, + { + Input: Input{Metric: "metric2", Hash: "1001", Ts: 1000, Value: 33}, + Output: Output{Metric: "metric2", Value: 0, Ok: false}, + }, + { + Input: Input{Metric: "metric2", Hash: "1001", Ts: 1000, Value: 40}, + Output: Output{Metric: "metric2", Value: 7, Ok: true}, + }, + } + + op := newActionOperator(ActionTypeDelta, nil, []string{"metric1", "metric2"}) + for _, c := range cases { + m, v, ok := op.GetOrUpdate(c.Input.Metric, c.Input.Hash, c.Input.Ts, c.Input.Value) + assert.Equal(t, c.Output.Metric, m) + assert.Equal(t, c.Output.Value, v) + assert.Equal(t, c.Output.Ok, ok) + } +} + +func TestActionRate(t *testing.T) { + type Input struct { + Metric string + Hash string + Ts int64 + Value float64 + } + type Output struct { + Metric string + Value float64 + Ok bool + } + + cases := []struct { + Input Input + Output Output + }{ + { + Input: Input{Metric: "metric3", Hash: "1001", Ts: 1000, Value: 10}, + Output: Output{Metric: "metric3", Value: 10, Ok: true}, + }, + { + Input: Input{Metric: "metric4", Hash: "1001", Ts: 1000, Value: 11}, + Output: Output{Metric: "metric4", Value: 11, Ok: true}, + }, + { + Input: Input{Metric: "metric1", Hash: "1001", Ts: 1000, Value: 11}, + Output: Output{Metric: "metric1.rate", Value: 0, Ok: false}, + }, + { + Input: Input{Metric: "metric1", Hash: "1001", Ts: 2000, Value: 22}, + Output: Output{Metric: "metric1.rate", Value: float64(11) / float64(1000), Ok: true}, + }, + { + Input: Input{Metric: "metric2", Hash: "1001", Ts: 1000, Value: 33}, + Output: Output{Metric: "metric2.rate", Value: 0, Ok: false}, + }, + { + Input: Input{Metric: "metric2", Hash: "1001", Ts: 2000, Value: 40}, + Output: Output{Metric: "metric2.rate", Value: float64(7) / float64(1000), Ok: true}, + }, + } + + rateOpts := []ActionRate{ + {Source: "metric1", Destination: "metric1.rate"}, + {Source: "metric2", Destination: "metric2.rate"}, + } + + op := newActionOperator(ActionTypeRate, rateOpts, nil) + for _, c := range cases { + m, v, ok := op.GetOrUpdate(c.Input.Metric, c.Input.Hash, c.Input.Ts, c.Input.Value) + assert.Equal(t, c.Output.Metric, m) + assert.Equal(t, c.Output.Value, v) + assert.Equal(t, c.Output.Ok, ok) + } +} diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go index bd8c452c9..4a6e7aea9 100644 --- a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector.go @@ -49,14 +49,11 @@ func init() { mb.Registry.MustAddMetricSet("prometheus", "collector", New, mb.WithHostParser(hostParser)) } -// MetricSet : type MetricSet struct { mb.BaseMetricSet httpClient *HTTPClient namespace string - - deltaKeys map[string]struct{} - lastDeltaMetrics map[string]map[string]float64 // map[metricName]map[hash]value + actionOp *actionOperator useTempFile bool tempFilePattern string @@ -96,15 +93,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } logger.Infof("base.metric.set config: %+v", config) - deltaKeys := map[string]struct{}{} - lastDeltaMetrics := make(map[string]map[string]float64) - for _, key := range config.DiffMetrics { - deltaKeys[key] = struct{}{} - lastDeltaMetrics[key] = make(map[string]float64) + promRelabels, actionConfigs, err := handleRelabels(config.MetricRelabelConfigs) + if err != nil { + logger.Errorf("handle relabels failed: %v", err) + return nil, err } var relabels []*relabel.Config - data, err := yaml.Marshal(config.MetricRelabelConfigs) + data, err := yaml.Marshal(promRelabels) if err != nil { logger.Errorf("marshal metric relabel config failed: %s", err) return nil, err @@ -131,12 +127,21 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + // 目前 delta/rate 为互斥，只能支持其一 + var actionOp *actionOperator + if len(config.DiffMetrics) > 0 { + actionOp = newActionOperator(ActionTypeDelta, nil, config.DiffMetrics) + } else if len(actionConfigs.Rate) > 0 { + actionOp = newActionOperator(ActionTypeRate, actionConfigs.Rate, nil) + } else if len(actionConfigs.Delta) > 0 { + actionOp = newActionOperator(ActionTypeDelta, nil, actionConfigs.Delta) + } + return &MetricSet{ BaseMetricSet: base, httpClient: httpClient, namespace: config.Namespace, - deltaKeys: deltaKeys, - lastDeltaMetrics: lastDeltaMetrics, + actionOp: actionOp, useTempFile: utils.HasTempDir(), tempFilePattern: config.TempFilePattern, remoteClient: &http.Client{Timeout: duration}, @@ -151,17 +156,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } -func (m *MetricSet) getEventFromPromEvent(promEvent *tasks.PromEvent) (common.MapStr, *diffKey) { +func (m *MetricSet) getEventFromPromEvent(promEvent *tasks.PromEvent) []common.MapStr { // 执行 relabels 规则 if len(m.MetricRelabelConfigs) != 0 { if !m.metricRelabel(promEvent) { - return nil, nil + return nil } } // 基于配置进行维度复制 - if m.DimensionReplace != nil { - m.replaceDimensions(promEvent) + if len(m.DimensionReplace) > 0 { + newDims := make(map[string]interface{}) + for k, v := range promEvent.Labels { + if targetKey, ok := m.DimensionReplace[k]; ok { + newDims[targetKey] = v + } + } + for k, v := range newDims { + promEvent.Labels[k] = v + } } // labels 处理 @@ -200,24 +213,30 @@ func (m *MetricSet) getEventFromPromEvent(promEvent *tasks.PromEvent) (common.Ma } } - // 差值计算 - var dk *diffKey - lines, ok := m.lastDeltaMetrics[promEvent.Key] - if ok { - currValue := promEvent.Value - lastValue, ok := lines[promEvent.HashKey] - if ok { - event["value"] = currValue - lastValue - } - dk = &diffKey{ - key: promEvent.Key, - hash: promEvent.HashKey, - value: currValue, - } - } else { + // 不需要额外的 action 操作 + if m.actionOp == nil { event["value"] = promEvent.Value + return []common.MapStr{event} + } + + newMetric, newValue, ok := m.actionOp.GetOrUpdate(promEvent.Key, promEvent.HashKey, promEvent.TS, promEvent.Value) + if !ok { + return nil + } + + // 不需要复制指标 + if newMetric == promEvent.Key { + event["value"] = newValue + return []common.MapStr{event} } - return event, dk + + // 需要复制指标 + event["value"] = promEvent.Value // 保留原有 value + + newEvent := event.Clone() + newEvent["key"] = newMetric + newEvent["value"] = newValue + return []common.MapStr{event, newEvent} } // getEventsFromFile 从文件获取指标 @@ -239,12 +258,6 @@ func (m *MetricSet) getEventsFromFile(fileName string) (<-chan common.MapStr, er return m.getEventsFromReader(f, cleanup, true), nil } -type diffKey struct { - key string - hash string - value float64 -} - // getEventsFromReader 从 reader 获取指标 func (m *MetricSet) getEventsFromReader(metricsReader io.ReadCloser, cleanup func(), up bool) <-chan common.MapStr { if m.MetricRelabelRemote != "" { @@ -272,9 +285,8 @@ func (m *MetricSet) getEventsFromReader(metricsReader io.ReadCloser, cleanup fun milliTs := time.Now().UnixMilli() eventChan := make(chan common.MapStr) - var total atomic.Int64 - // 补充 up 指标文本 + var total atomic.Int64 markUp := func(failed bool, t0 time.Time) { // 需要减去自监控指标 events := m.asEvents(CodeScrapeLine(int(total.Load()-2), m.logkvs()), milliTs) @@ -289,40 +301,34 @@ func (m *MetricSet) getEventsFromReader(metricsReader io.ReadCloser, cleanup fun } } + // 消费指标文本并生成事件 + var produceErr atomic.Bool + consume := func() { + for line := range linesCh { + events, err := m.produceEvents(line, milliTs) + if err != nil { + logger.Warnf("failed to produce events: %v", err) + produceErr.Store(true) + continue + } + for j := 0; j < len(events); j++ { + eventChan <- events[j] + total.Add(1) + } + } + } + start := time.Now() go func() { defer close(eventChan) defer cleanup() - var lastDiffMetricMut sync.Mutex - lastDiffMetrics := make(map[string]map[string]float64) - for key := range m.deltaKeys { - lastDiffMetrics[key] = make(map[string]float64) - } - wg := sync.WaitGroup{} - var produceErr atomic.Bool for i := 0; i < worker; i++ { wg.Add(1) go func() { defer wg.Done() - for line := range linesCh { - events, dk, err := m.produceEvents(line, milliTs) - if err != nil { - logger.Warnf("failed to produce events: %v", err) - produceErr.Store(true) - continue - } - if dk != nil { - lastDiffMetricMut.Lock() - lastDiffMetrics[dk.key][dk.hash] = dk.value - lastDiffMetricMut.Unlock() - } - for j := 0; j < len(events); j++ { - eventChan <- events[j] - total.Add(1) - } - } + consume() }() } wg.Wait() @@ -330,7 +336,6 @@ func (m *MetricSet) getEventsFromReader(metricsReader io.ReadCloser, cleanup fun if up { markUp(produceErr.Load(), start) // 一次采集只上报一次状态 } - m.lastDeltaMetrics = lastDiffMetrics }() return eventChan } @@ -339,21 +344,43 @@ func normalizeName(s string) string { return strings.Join(strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' }), "_") } +func keyFunc(m common.MapStr) string { + objKey, err := m.GetValue("key") + if err != nil { + return "" + } + + s, ok := objKey.(string) + if !ok { + return "" + } + return s +} + func (m *MetricSet) asEvents(line string, timestamp int64) []common.MapStr { - events, _, _ := m.produceEvents(line, timestamp) + events, _ := m.produceEvents(line, timestamp) return events } -func (m *MetricSet) produceEvents(line string, timestamp int64) ([]common.MapStr, *diffKey, error) { +func (m *MetricSet) produceEvents(line string, timestamp int64) ([]common.MapStr, error) { if len(line) <= 0 || line[0] == '#' { - return nil, nil, nil + return nil, nil } timeOffset := 24 * time.Hour * 365 * 2 // 默认可容忍偏移时间为两年 tsHandler, _ := tasks.GetTimestampHandler("s") - promEvent, err := tasks.NewPromEvent(line, timestamp, timeOffset, tsHandler) + + var promEvent tasks.PromEvent + var err error + // fastpath: 如果不需要执行额外的 action 则代表不需要计算 hash + if m.actionOp == nil { + promEvent, err = tasks.NewPromEventFast(line, timestamp, timeOffset, tsHandler) + } else { + promEvent, err = tasks.NewPromEvent(line, timestamp, timeOffset, tsHandler) + } + if err != nil { - return nil, nil, errors.Wrapf(err, "parse line(%s) failed", line) + return nil, errors.Wrapf(err, "parse line(%s) failed", line) } if m.normalizeMetricName { @@ -361,21 +388,37 @@ func (m *MetricSet) produceEvents(line string, timestamp int64) ([]common.MapStr } // 生成事件 - var events []common.MapStr - event, dk := m.getEventFromPromEvent(&promEvent) - if event == nil { - return nil, nil, nil + events := m.getEventFromPromEvent(&promEvent) + if len(events) == 0 { + return nil, nil + } + + // 不需要指标复制 流程结束 + if len(m.MetricReplace) == 0 { + return events, nil } - events = append(events, event) - // 基于配置进行指标复制 - targetMetricKey := m.getTargetMetricKey(&promEvent) - if targetMetricKey != "" { + var cloneEvents []common.MapStr + for i := 0; i < len(events); i++ { + event := events[i] + key := keyFunc(event) + + // 没找 key 或者 key 不需要复制则跳过 + if len(key) == 0 { + continue + } + targetKey, ok := m.MetricReplace[key] + if !ok { + continue + } + targetEvent := event.Clone() - targetEvent["key"] = targetMetricKey - events = append(events, targetEvent) + targetEvent["key"] = targetKey + cloneEvents = append(cloneEvents, targetEvent) } - return events, dk, nil + + events = append(events, cloneEvents...) + return events, nil } func (m *MetricSet) logkvs() []define.LogKV { @@ -429,6 +472,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { return summary, err } + // 将自监控指标当成普通指标文本处理 metricsFile.WriteString("\n" + CodeScrapeSize(int(info.Size()), m.logkvs())) metricsFile.WriteString("\n" + CodeScrapeDuration(time.Since(startTime).Seconds(), m.logkvs())) diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go index fd7006f03..ec9082bd2 100644 --- a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/collector_test.go @@ -20,16 +20,7 @@ import ( ) func TestGetEventFromPromEvent(t *testing.T) { - deltaKeys := map[string]struct{}{} - lastDeltaMetrics := make(map[string]map[string]float64) - for _, key := range []string{"metric1", "metric2"} { - deltaKeys[key] = struct{}{} - lastDeltaMetrics[key] = make(map[string]float64) - } - mb := &MetricSet{ - deltaKeys: deltaKeys, - lastDeltaMetrics: lastDeltaMetrics, normalizeMetricName: true, } diff --git a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/relabel.go b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/relabel.go index bddd4d710..8a6d00da0 100644 --- a/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/relabel.go +++ b/pkg/bkmonitorbeat/tasks/metricbeat/module/prometheus/collector/relabel.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" + "gopkg.in/yaml.v2" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/bkmonitorbeat/tasks" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" @@ -118,27 +119,61 @@ func (m *MetricSet) metricRelabel(promEvent *tasks.PromEvent) bool { } if promEvent.Key == "" { - logger.Errorf("missing metric key when collect metric: %s", promEvent.Key) return false } return true } -func (m *MetricSet) replaceDimensions(promEvent *tasks.PromEvent) { - for sourceKey, sourceValue := range promEvent.Labels { - if targetKey, ok := m.DimensionReplace[sourceKey]; ok { - logger.Debugf("copy dimension: %s to %s in metric: %s", sourceKey, targetKey, promEvent.Key) - promEvent.Labels[targetKey] = sourceValue - } - } +// promRelabel prometheus 提供的内置 relabels 配置 +type promRelabel struct { + // A list of labels from which values are taken and concatenated + // with the configured separator in order. + SourceLabels []string `yaml:"source_labels,flow,omitempty"` + // Separator is the string between concatenated values from the source labels. + Separator string `yaml:"separator,omitempty"` + // Regex against which the concatenation is matched. + Regex string `yaml:"regex,omitempty"` + // Modulus to take of the hash of concatenated values from the source labels. + Modulus uint64 `yaml:"modulus,omitempty"` + // TargetLabel is the label to which the resulting string is written in a replacement. + // Regexp interpolation is allowed for the replace action. + TargetLabel string `yaml:"target_label,omitempty"` + // Replacement is the regex replacement pattern to be used. + Replacement string `yaml:"replacement,omitempty"` + // Action is the action to be performed for the relabeling. + Action string `yaml:"action,omitempty"` } -func (m *MetricSet) getTargetMetricKey(promEvent *tasks.PromEvent) string { - if m.MetricReplace != nil { - if targetKey, ok := m.MetricReplace[promEvent.Key]; ok { - logger.Debugf("copy metric: %s to %s", promEvent.Key, targetKey) - return targetKey +func handleRelabels(configs interface{}) ([]promRelabel, *ActionConfigs, error) { + var relabels []promRelabel + data, err := yaml.Marshal(configs) + if err != nil { + return nil, nil, err + } + if err = yaml.Unmarshal(data, &relabels); err != nil { + return nil, nil, err + } + + dst := make([]promRelabel, 0) + ac := &ActionConfigs{} + + for i := 0; i < len(relabels); i++ { + rl := relabels[i] + switch rl.Action { + case ActionTypeRate: + if len(rl.SourceLabels) == 1 && rl.SourceLabels[0] == model.MetricNameLabel { + ac.Rate = append(ac.Rate, ActionRate{ + Source: rl.Regex, // 借用 Regex 字段，实际并不支持正则 + Destination: rl.Replacement, + }) + } + + case ActionTypeDelta: + ac.Delta = append(ac.Delta, rl.SourceLabels...) + + default: // prometheus 标准 actions + dst = append(dst, rl) } } - return "" + return dst, ac, nil } diff --git a/pkg/bkmonitorbeat/tasks/prom_event.go b/pkg/bkmonitorbeat/tasks/prom_event.go index 7308a5886..b1ced19df 100644 --- a/pkg/bkmonitorbeat/tasks/prom_event.go +++ b/pkg/bkmonitorbeat/tasks/prom_event.go @@ -27,12 +27,14 @@ import ( "github.com/prometheus/prometheus/model/textparse" ) +var ErrNaN = errors.New("value is NaN or Inf") + // PromEvent store the lines prometheus data type PromEvent struct { Key string Value float64 Labels common.MapStr - AggreValue common.MapStr + AggValue common.MapStr DimensionString string // ordered dimension string HashKey string TS int64 @@ -47,9 +49,9 @@ func (pe *PromEvent) exemplarString() string { return HashLabels(pe.Exemplar.Labels) } -// GetAggreValue get same timestamp and same dimension metrics -func (pe *PromEvent) GetAggreValue() common.MapStr { - return pe.AggreValue +// GetAggValue get same timestamp and same dimension metrics +func (pe *PromEvent) GetAggValue() common.MapStr { + return pe.AggValue } // GetLabels get dimensions @@ -98,15 +100,35 @@ func hashLabels(lbs labels.Labels) uint64 { // NewPromEvent 优先使用 V2 作为解析方式 出错再回退到 V1 还出错的话就抛出异常 func NewPromEvent(line string, ts int64, offsetTime time.Duration, handler TimestampHandler) (PromEvent, error) { - pe, err := NewPromEventV2(line, ts, offsetTime, handler) + pe, err := newPromEventV2(line, ts, offsetTime, handler, true) + if err == nil { + return pe, nil + } + + // NaN 的话就不再尝试了 + if errors.Is(err, ErrNaN) { + return pe, err + } + + return newPromEventV1(line, ts, offsetTime, handler, true) +} + +// NewPromEventFast 同 NewPromEvent 但不计算 PromEvent 哈希值 +func NewPromEventFast(line string, ts int64, offsetTime time.Duration, handler TimestampHandler) (PromEvent, error) { + pe, err := newPromEventV2(line, ts, offsetTime, handler, false) if err == nil { return pe, nil } - return NewPromEventV1(line, ts, offsetTime, handler) + // NaN 的话就不再尝试了 + if errors.Is(err, ErrNaN) { + return pe, err + } + + return newPromEventV1(line, ts, offsetTime, handler, false) } -func NewPromEventV2(line string, ts int64, offsetTime time.Duration, handler TimestampHandler) (PromEvent, error) { +func newPromEventV2(line string, ts int64, offsetTime time.Duration, handler TimestampHandler, hashKey bool) (PromEvent, error) { // \n 为解析分隔符 if !strings.HasSuffix(line, "\n") { line = line + "\n" @@ -123,7 +145,7 @@ func NewPromEventV2(line string, ts int64, offsetTime time.Duration, handler Tim case textparse.EntrySeries: _, timestamp, val := parser.Series() if math.IsInf(val, 0) || math.IsNaN(val) { - return pe, errors.New("value is NaN or Inf") + return pe, ErrNaN } var lbs labels.Labels @@ -154,17 +176,20 @@ func NewPromEventV2(line string, ts int64, offsetTime time.Duration, handler Tim pe.Key = lbs.Get("__name__") pe.Value = val pe.TS = peTs - pe.AggreValue = common.MapStr{} + pe.AggValue = common.MapStr{} - // 排序 dimensions - pe.DimensionString = HashLabels(newLbs) + if hashKey { + pe.DimensionString = HashLabels(newLbs) // 排序 dimensions + } } - pe.ProduceHashKey() + if hashKey { + pe.ProduceHashKey() + } return pe, nil } -func NewPromEventV1(line string, ts int64, offsetTime time.Duration, handler TimestampHandler) (PromEvent, error) { +func newPromEventV1(line string, ts int64, offsetTime time.Duration, handler TimestampHandler, hashKey bool) (PromEvent, error) { if !strings.HasSuffix(line, "\n") { line = line + "\n" } @@ -202,18 +227,21 @@ func NewPromEventV1(line string, ts int64, offsetTime time.Duration, handler Tim } if math.IsInf(value, 0) || math.IsNaN(value) { - return pe, errors.New("value is NaN or Inf") + return pe, ErrNaN } pe = PromEvent{ - Key: familyName, - Value: value, - Labels: labelMap, - AggreValue: common.MapStr{}, - TS: timestamp, - } - pe.DimensionString = HashLabels(newLbs) - pe.ProduceHashKey() + Key: familyName, + Value: value, + Labels: labelMap, + AggValue: common.MapStr{}, + TS: timestamp, + } + + if hashKey { + pe.DimensionString = HashLabels(newLbs) + pe.ProduceHashKey() + } return pe, nil } diff --git a/pkg/bkmonitorbeat/tasks/script/formatdata.go b/pkg/bkmonitorbeat/tasks/script/formatdata.go index f4e544356..fad98b8c6 100644 --- a/pkg/bkmonitorbeat/tasks/script/formatdata.go +++ b/pkg/bkmonitorbeat/tasks/script/formatdata.go @@ -41,12 +41,12 @@ func FormatOutput(out []byte, ts int64, offsetTime time.Duration, handler tasks. continue } - promEvent.AggreValue[promEvent.Key] = promEvent.Value + promEvent.AggValue[promEvent.Key] = promEvent.Value subRst, tsExist := aggRst[promEvent.TS] if tsExist { p, dmExist := subRst[promEvent.HashKey] if dmExist { - p.AggreValue[promEvent.Key] = promEvent.Value + p.AggValue[promEvent.Key] = promEvent.Value subRst[promEvent.HashKey] = p } else { subRst[promEvent.HashKey] = promEvent diff --git a/pkg/bkmonitorbeat/tasks/script/formatdata_test.go b/pkg/bkmonitorbeat/tasks/script/formatdata_test.go index 311e716e0..08d7a4710 100644 --- a/pkg/bkmonitorbeat/tasks/script/formatdata_test.go +++ b/pkg/bkmonitorbeat/tasks/script/formatdata_test.go @@ -42,10 +42,10 @@ func (s *ScriptFormatSuite) TestFormat() { s.Equal(now.Unix(), timestamp) s.Equal(len(eventData), 1) for _, pe := range eventData { - s.Equal(len(pe.GetAggreValue()), 1) + s.Equal(len(pe.GetAggValue()), 1) s.Equal(len(pe.GetLabels()), 1) s.Equal(pe.GetLabels()["mountpoint"], "/usr/local") - s.Equal(pe.GetAggreValue()["sys_disk_size"], 8.52597957704355) + s.Equal(pe.GetAggValue()["sys_disk_size"], 8.52597957704355) } } } @@ -56,7 +56,7 @@ func (s *ScriptFormatSuite) TestNewProm() { s.Equal(err, nil) pe, err := tasks.NewPromEvent(testData, 123, 365*24*time.Hour*10, handler) s.Equal(err, nil) - value := pe.GetAggreValue() + value := pe.GetAggValue() s.Equal(len(value), 0) labels := pe.GetLabels() s.Equal(len(labels), 2) @@ -71,7 +71,7 @@ func (s *ScriptFormatSuite) TestTimeStampProm() { ts := now.UnixMilli() pe, err := tasks.NewPromEvent(testData, ts, 365*24*time.Hour, handler) s.Equal(err, nil) - value := pe.GetAggreValue() + value := pe.GetAggValue() s.Equal(len(value), 0) labels := pe.GetLabels() s.Equal(len(labels), 2) diff --git a/pkg/bkmonitorbeat/tasks/script/gather.go b/pkg/bkmonitorbeat/tasks/script/gather.go index 3d7f35d3d..86428a3b0 100644 --- a/pkg/bkmonitorbeat/tasks/script/gather.go +++ b/pkg/bkmonitorbeat/tasks/script/gather.go @@ -95,7 +95,7 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { ev.UTCTime = originEvent.UTCTime ev.UserTime = time.Unix(ev.Timestamp, 0).UTC().Format(bkcommon.TimeFormat) - for aggKey, aggValue := range pe.AggreValue { + for aggKey, aggValue := range pe.AggValue { ev.Metric[aggKey] = aggValue } if len(pe.Labels) > 0 { @@ -163,7 +163,7 @@ func (g *Gather) KeepOneDimension(data map[int64]map[string]tasks.PromEvent) { dimFieldNames = append(dimFieldNames, "") // 先占个空位 newAggValue := make(common.MapStr) - for aggKey, aggValue := range pe.AggreValue { + for aggKey, aggValue := range pe.AggValue { dimFieldNames[dimensionNamesLen] = aggKey hashKey := utils.GeneratorHashKey(dimFieldNames) if !keySet.Has(hashKey) { @@ -171,7 +171,7 @@ func (g *Gather) KeepOneDimension(data map[int64]map[string]tasks.PromEvent) { newAggValue[aggKey] = aggValue } } - pe.AggreValue = newAggValue + pe.AggValue = newAggValue // 如果该维度下的还有指标未被清理。则保留这个维度的数据 if len(newAggValue) > 0 { diff --git a/pkg/bkmonitorbeat/tasks/tcp/gather.go b/pkg/bkmonitorbeat/tasks/tcp/gather.go index b330cd80b..eb9e790c5 100644 --- a/pkg/bkmonitorbeat/tasks/tcp/gather.go +++ b/pkg/bkmonitorbeat/tasks/tcp/gather.go @@ -132,25 +132,16 @@ func (g *Gather) checkTargetHost(ctx context.Context, taskConf *configs.TCPTaskC } func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { - hosts := make([]string, 0) resultMap := make(map[string][]string) taskConf := g.TaskConfig.(*configs.TCPTaskConfig) g.PreRun(ctx) defer g.PostRun(ctx) - if taskConf.TargetHost == "" && len(taskConf.TargetHostList) == 0 { - // 不上报任何数据 - logger.Debugf("tcp TargetHostList is empty.") + hosts := taskConf.Hosts() + if len(hosts) == 0 { return } - // 获取配置的host列表 - if taskConf.TargetHost != "" { - hosts = append(hosts, taskConf.TargetHost) - } - if len(taskConf.TargetHostList) > 0 { - hosts = taskConf.TargetHostList - } hostsInfo := tasks.GetHostsInfo(ctx, hosts, taskConf.DNSCheckMode, taskConf.TargetIPType, configs.Tcp) for _, h := range hostsInfo { if h.Errno != define.CodeOK { @@ -165,18 +156,16 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { // 解析目标为ip列表 var wg sync.WaitGroup for taskHost, result := range resultMap { - // 循环列表检测 for _, targetHost := range result { - // 获取并发限制信号量 err := g.GetSemaphore().Acquire(ctx, 1) if err != nil { - logger.Errorf("Semaphore Acquire failed for task tcp task id: %d", g.TaskConfig.GetTaskID()) + logger.Errorf("task(%d) semaphore acquire failed", g.TaskConfig.GetTaskID()) return } + wg.Add(1) go func(tHost, host string) { - // 初始化事件 event := g.newEvent(taskConf, tHost) event.ResolvedIP = host @@ -198,12 +187,10 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { wg.Wait() } -// New : func New(globalConfig define.Config, taskConfig define.TaskConfig) define.Task { gather := &Gather{} gather.GlobalConfig = globalConfig gather.TaskConfig = taskConfig - gather.Init() return gather diff --git a/pkg/bkmonitorbeat/tasks/udp/gather.go b/pkg/bkmonitorbeat/tasks/udp/gather.go index cb10b35fd..07edcd036 100644 --- a/pkg/bkmonitorbeat/tasks/udp/gather.go +++ b/pkg/bkmonitorbeat/tasks/udp/gather.go @@ -145,23 +145,14 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { ctx, cancel := context.WithTimeout(ctx, g.config.Timeout) defer cancel() - hosts := make([]string, 0) resultMap := make(map[string][]string) start := time.Now() - if g.config.TargetHost == "" && len(g.config.TargetHostList) == 0 { - // 不上报任何数据 - logger.Debugf("udp TargetHostList is empty.") + hosts := g.config.Hosts() + if len(hosts) == 0 { return } - // 获取配置的host列表 - if g.config.TargetHost != "" { - hosts = append(hosts, g.config.TargetHost) - } - if len(g.config.TargetHostList) > 0 { - hosts = g.config.TargetHostList - } hostsInfo := tasks.GetHostsInfo(ctx, hosts, g.config.DNSCheckMode, g.config.TargetIPType, configs.Udp) for _, h := range hostsInfo { if h.Errno != define.CodeOK { @@ -177,14 +168,14 @@ func (g *Gather) Run(ctx context.Context, e chan<- define.Event) { // 解析目标为ip列表 var wg sync.WaitGroup for taskHost, result := range resultMap { - // 循环列表检测 for _, targetHost := range result { // 获取并发限制信号量 err := g.GetSemaphore().Acquire(ctx, 1) if err != nil { - logger.Errorf("Semaphore Acquire failed for task udp task id: %d", g.TaskConfig.GetTaskID()) + logger.Errorf("task(%d) semaphore acquire failed", g.TaskConfig.GetTaskID()) return } + wg.Add(1) go func(tHost, host string) { // 初始化事件 @@ -275,7 +266,6 @@ func (g *Gather) detect(conn net.Conn) ([]byte, define.NamedCode) { return body[:n], define.CodeOK } -// New : func New(globalConfig define.Config, taskConfig define.TaskConfig) define.Task { gather := &Gather{} gather.GlobalConfig = globalConfig diff --git a/pkg/operator/README.md b/pkg/operator/README.md index 81dd96920..2dc67e84f 100644 --- a/pkg/operator/README.md +++ b/pkg/operator/README.md @@ -20,6 +20,8 @@ bkmonitor-operator 整体架构如下： * k8seventdataid：内置事件 dataid。 * k8smetricdataid：内置指标 dataid。 +请参考 [dataid 介绍](docs/dataId.md) 文档。 + 为了管理区分，operator 注入的 serviceMonitor 采集会使用内置 dataid，这个是通过在 `metadata.annotations` 声明 `isSystem: "true"` 指定的。而用户的数据上报会用自定义 dataid。当然，我们也对 dataid 进行了扩展，用户可以在监控这边申请新的 dataid，用于匹配特性维度的 serviceMonitor，以便对数据进行隔离，如果量太大的话，我们会对 dataid 进行存储集群的划分。 在集群接入后，dataid 资源就绪后，operator 就会开始工作了。operator 涉及到多个组件，这里一一做介绍： diff --git a/pkg/operator/common/eplabels/labels.go b/pkg/operator/common/eplabels/labels.go new file mode 100644 index 000000000..9d0bfb1cd --- /dev/null +++ b/pkg/operator/common/eplabels/labels.go @@ -0,0 +1,38 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package eplabels + +func EndpointNodeName(endpointslice bool) string { + if endpointslice { + return "__meta_kubernetes_endpointslice_node_name" + } + return "__meta_kubernetes_endpoint_node_name" +} + +func EndpointAddressTargetKind(endpointslice bool) string { + if endpointslice { + return "__meta_kubernetes_endpointslice_address_target_kind" + } + return "__meta_kubernetes_endpoint_address_target_kind" +} + +func EndpointAddressTargetName(endpointslice bool) string { + if endpointslice { + return "__meta_kubernetes_endpointslice_address_target_name" + } + return "__meta_kubernetes_endpoint_address_target_name" +} + +func EndpointPortName(endpointslice bool) string { + if endpointslice { + return "__meta_kubernetes_endpointslice_port_name" + } + return "__meta_kubernetes_endpoint_port_name" +} diff --git a/pkg/operator/docs/API.md b/pkg/operator/docs/API.md index 84656e960..67853e48b 100644 --- a/pkg/operator/docs/API.md +++ b/pkg/operator/docs/API.md @@ -23,28 +23,6 @@ * GET /cluster_info: 查看集群信息 -### 工作负载 - -* GET /workload: 查看所有 Pod 工作负载 -* GET /workload/node/{node}: 查看指定主机的 Pod 工作负载 - -### 故障定位 - -* GET /check: 故障排查接口，支持 `monitor` 关键字查询参数 - - ```shell - $ curl http://localhost:8080/check?monitor=blueking - ``` -* GET /check/dataid: 检查 dataid -* GET /check/scrape: 检查采集任务指标数量 -* GET /check/scrape/{namespace}: 检查某个 namespace 指标文本并返回 -* GET /check/scrape/{namespace}/{monitor}: 检查某个 namespace 下的 monitor 指标文本并返回 -* GET /check/namespace: 检查黑白名单配置 -* GET /check/active_discover: 检查活跃的 discover 情况 -* GET /check/active_child_config: 检查活跃的采集任务情况 -* GET /check/active_shared_discovery: 检查活跃的 shared_discovery 情况 -* GET /check/monitor_resource: 检查监控资源 - ### 性能分析 * GET /debug/pprof/snapshot: 下载 profiles 快照 @@ -56,3 +34,7 @@ * GET /debug/pprof/symbol: symbol 采集 * GET /debug/pprof/trace: trace 采集 * GET /debug/pprof/{other}: 其他 profile 项采集 + +### 故障分析 + +请参考 [check-help](./check-help.md) 文档。 diff --git a/pkg/operator/docs/check-help.md b/pkg/operator/docs/check-help.md new file mode 100644 index 000000000..3047b284f --- /dev/null +++ b/pkg/operator/docs/check-help.md @@ -0,0 +1,442 @@ +# 故障定位 API 帮助文档 + +在了解故障定位 API 之前，可以先来查看一下所有的路由信息。 +```shell +$ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080 + +# Admin Routes +-------------- +* POST /-/logger +* POST /-/reload +* POST /-/dispatch + +# Metadata Routes +----------------- +* GET /metrics +* GET /version +* GET /cluster_info +* GET /workload +* GET /workload/node/{node} +* GET /relation/metrics +* GET /rule/metrics + +# Check Routes +-------------- +* GET /check?monitor=${monitor}&scrape=true|false +* GET /check/dataid +* GET /check/scrape +* GET /check/scrape/{namespace} +* GET /check/scrape/{namespace}/{monitor} +* GET /check/namespace +* GET /check/monitor_blacklist +* GET /check/active_discover +* GET /check/active_child_config +* GET /check/active_shared_discovery +* GET /check/monitor_resource + +# Profile Routes +---------------- +* GET /debug/pprof/snapshot +* GET /debug/pprof/cmdline +* GET /debug/pprof/profile +* GET /debug/pprof/symbol +* GET /debug/pprof/trace +* GET /debug/pprof/{other} +``` +operator 提供了 Admin、Metadata、Check、Profile 相关的路由。 +- Admin：提供了重新加载配置等接口。 +- MetaData：提供了查看自定义指标、版本、工作负载等接口。 +- Profile：提供了性能分析等接口。 + +本文档将重点介绍 Check Routes。 + +## 故障排查 API 以及用法 + +### GET /check + +> 故障排查接口，支持 `monitor` 关键字查询参数。 + +在进行故障排查之前可以先使用 /check 用于来查询相关信息看一下是否能匹配上。例如版本信息、DataID 信息、集群信息、operator 监听 Endpoints 的数量等等，确定相关信息是否符合预期。 + +如若发现与预期不一致，说明没有监控到相关资源。可能是配置错误或者鉴权问题。 + +```shell +# bkm-operator-79486746f5-n6ztd 为当前集群中 operator pod。 +# 可以通过 `kubectl get pods -n bkmonitor-operator | grep bkm-operator` 查找。 +# 监听端口默认为 8080。 + +$ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/check?monitor=blueking + +[√] check kubernetes version +- Description: kubernetes 集群版本为 v1.20.6-tke.34 + +[√] check operator version +- Description: bkmonitor-operator 版本信息 +{ + "version": "3.6.2187", + "git_hash": "0b10d1a9be30978b30c4ccd1e089dea2922e5276", + "build_time": "2024-08-05_06:22:37上午" +} + +[√] check dataids +- Description: 期待 dataids 数量应大于等于 3 个，目前发现 3 个 +[ + { + "dataid": 1572864, + "name": "k8smetricdataid", + "labels": { + "isCommon": "true", + "isSystem": "true", + "usage": "metric" + } + }, + { + "dataid": 1572865, + "name": "custommetricdataid", + "labels": { + "isCommon": "true", + "isSystem": "false", + "usage": "metric" + } + }, + { + "dataid": 1572866, + "name": "k8seventdataid", + "labels": { + "isCommon": "true", + "isSystem": "true", + "usage": "event" + } + } +] + +[√] check cluster information +- Description: 集群信息 +{ + "bcs_cluster_id": "BCS-K8S-00000", + "bizid": "2", + "bk_env": "" +} + +[√] check endpoint +- Description: operator 监听 monitor endpoints 数量，共 138 个 +{ + "ServiceMonitor/aiops-default/kg-dgraph-kg-dgraph-alpha/0": 1, + "ServiceMonitor/aiops-default/kg-dgraph-kg-dgraph-zero/0": 1, + "ServiceMonitor/bkbase/bkbase-dgraph-bkbase-dgr-alpha/0": 3, + "ServiceMonitor/bkbase/bkbase-dgraph-bkbase-dgr-zero/0": 3, + "ServiceMonitor/bkbase/bkbase-jobnavischeduler/0": 1, + "ServiceMonitor/bkbase/bkbase-pulsar-broker-http/0": 1, + "ServiceMonitor/bkbase/bkbase-pulsar-broker-pulsar/0": 1, + "ServiceMonitor/bkbase/bkbase-queryengine-api-servicemonitor/0": 1, + "ServiceMonitor/bkbase/bkbase-querymanager-api-servicemonitor/0": 1, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-coredns/0": 2, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-kube-proxy/0": 24, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-kube-state-metrics/0": 1, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-kubelet/0": 24, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-kubelet/1": 24, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-kubelet/2": 24, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-bkmonit-node-exporter/0": 24, + "ServiceMonitor/bkmonitor-operator/bkmonitor-operator-operator/0": 1, + "ServiceMonitor/blueking/bk-apigateway-apigateway/0": 1 +}, +[√] check nodes +- Description: 获取集群节点列表成功，节点数量为 24，最近一次更新时间 2024-08-08T07:17:21Z + +[√] check kubernetes secrets handled +- Description: 操作 secrets 资源未出现错误，最近一次操作时间 2024-08-19T02:17:27Z + +[√] check monitor resources +- Description: 通过 'blueking' 关键字匹配到以下监控资源。 +* 监测到 ServiceMonitor/PodMonitor/Probe 资源以及对应的采集目标，请检查资源数量是否一致 +[ + { + "kind": "ServiceMonitor", + "namespace": "blueking", + "name": "bk-apigateway-apigateway", + "index": 0, + "count": 1, + "location": [ + { + "address": "127.0.0.1:6008", + "node": "127.0.0.1", + "target": "http://127.0.0.1:6008/metrics", + "dataid": 1572865 + } + ] + } +] +* 生成的 bkmonitorbeat 采集配置文件 +[ + { + "service": "ServiceMonitor/blueking/bk-apigateway-apigateway/0", + "dataid": 1572865, + "filename": "10-0-7-53-172-17-7-74-6008-metrics-15502861147355311619-0", + "node": "127.0.0.2" + } +] +... +``` +在检查信息时，我们可以重点观察上面的 `check monitor resources`。上面的例子中使用了关键字 `blueking` 进行了过滤。 + +**关键字匹配均为 Contains 模糊匹配，非精确匹配。** + +当我们想根据 serviceMonitor/podMonitor 名称或者 namespace 等等相关信息做过滤，可以使用关键字。 + +* 对于`监测到 ServiceMonitor/PodMonitor/Probe 资源以及对应的采集目标`是以 serviceMonitor/podMonitor 名称、serviceMonitor/podMonitor 的 namespace 进行匹配的。 + +* 对于`生成的 bkmonitorbeat 的采集配置文件` 是以 MonitorMeta 的 Kind/Namespace/Name/Index 的格式来进行匹配的。MonitorMeta 描述了监控类型的元数据信息，目前类型有 serviceMonitor, podMonitor, probe。 + +下面是关于 `check monitor resources` 返回数据的说明。 + +**1）监测到 ServiceMonitor/PodMonitor/Probe 资源以及对应的采集目标** + +当想知道每个 serviceMonitor/podMonitor 监控的具体信息可以使用这个接口，serviceMonitor/podMonitor 会配置抓取若干个端口的信息，如 path、port 等信息。而一个 serviceMonitor 会监控到若干 service，每个 serivce 的 Endpoints 是一系列的 IP:Port。 + +**2）生成的 bkmonitorbeat 采集配置文件** + +`child_config` 是由蓝鲸监控定义的采集器采集配置文件。/check 接口可以查看 serviceMonitor/podMonitor 匹配到哪些采集任务以及采集任务被分配到了哪个 node 上、以及对应的 DataID。 + +这里我们简单举个例子，介绍一下 serviceMonitor 的匹配规则。 + +* 先查询 namespace 下的所有 serviceMonitor + ```shell + $ kubectl get servicemonitors.monitoring.coreos.com -n bkmonitor-operator + NAME AGE + bkmonitor-operator-bkmonit-apiserver 397d + bkmonitor-operator-bkmonit-coredns 397d + bkmonitor-operator-bkmonit-kube-controller-manager 397d + bkmonitor-operator-bkmonit-kube-proxy 397d + bkmonitor-operator-bkmonit-kube-state-metrics 397d + bkmonitor-operator-bkmonit-kubelet 397d + bkmonitor-operator-bkmonit-node-exporter 397d + bkmonitor-operator-operator 208d + ``` + +* 查看其中某一个 serviceMonitor 信息，以 bkmonitor-operator-bkmonit-kubelet 为例。 + ```shell + $ kubectl get servicemonitors.monitoring.coreos.com -n bkmonitor-operator bkmonitor-operator-bkmonit-kubelet -oyaml + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + name: bkmonitor-operator-bkmonit-kubelet + namespace: bkmonitor-operator + spec: + endpoints: + ... + ... + jobLabel: k8s-app + namespaceSelector: + matchNames: + - bkmonitor-operator + # servicemonitor 匹配规则 + selector: + matchLabels: + app.kubernetes.io/managed-by: bkmonitor-operator + k8s-app: kubelet + ``` + serviceMonitor 的 selector 和 namespaceSelector 字段用于匹配 Service 资源。 + +* 根据查询 serviceMonitor 匹配的 Service + ```shell + $ kubectl get service -n bkmonitor-operator -l app.kubernetes.io/managed-by=bkmonitor-operator,k8s-app=kubelet + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + bkmonitor-operator-kubelet ClusterIP None 10250/TCP,10255/TCP,4194/TCP 397d + ``` + bkmonitor-operator-bkmonit-kubelet 匹配到的 Service 为 bkmonitor-operator-kubelet。 + +* 查询 Service 对应的 Endpoints + ```shell + $ kubectl describe service -n bkmonitor-operator bkmonitor-operator-kubelet + Name: bkmonitor-operator-kubelet + Namespace: bkmonitor-operator + Labels: app.kubernetes.io/managed-by=bkmonitor-operator + app.kubernetes.io/name=kubelet + k8s-app=kubelet + Annotations: + Selector: + Type: ClusterIP + IP Families: + IP: None + IPs: None + Port: https-metrics 10250/TCP + TargetPort: 10250/TCP + Endpoints: 127.0.3.8:10250,127.0.4.6:10250,127.0.4.8:10250 + 21 more... + Port: http-metrics 10255/TCP + TargetPort: 10255/TCP + Endpoints: 127.0.3.8:10255,127.0.4.6:10255,127.0.4.8:10255 + 21 more... + Port: cadvisor 4194/TCP + TargetPort: 4194/TCP + Endpoints: 127.0.3.8:4194,127.0.4.6:4194,127.0.4.8:4194 + 21 more... + Session Affinity: None + ``` + 这里的 Endpoints 即例子中 bkmonitor-operator-bkmonit-kubelet 这个 serviceMonitor 匹配的 IP:Port 列表。 + +接下来再来看看 podMonitor 的匹配规则 + +* 查询 namespace 下所有的 podMonitor + ```shell + $ kubectl get podMonitor frost-podmonitor-test -n bkmonitor-operator -oyaml + NAME AGE + frost-podmonitor-test 167m + ``` + +* 查询 podMonitor 的配置信息 + ```shell + $ kubectl get podMonitor frost-podmonitor-test -n bkmonitor-operator -oyaml + apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: frost-podmonitor-test + namespace: bkmonitor-operator + spec: + namespaceSelector: + matchNames: + - bkmonitor-operator + podMetricsEndpoints: + - interval: 15s + path: /metrics + port: http + # podMonitor 匹配规则 + selector: + matchLabels: + app.kubernetes.io/bk-component: bkmonitor-operator + ``` + +* 查询 podMonitor 匹配的 pod + ```shell + kubectl get pod -n bkmonitor-operator -l app.kubernetes.io/bk-component=bkmonitor-operator -owide + NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES + bkm-operator-9964ccb66-485cs 1/1 Running 0 32m 127.1.11.11 127.0.7.101 + ``` + podMonitor 匹配到的 pod 就是 bkm-operator-9964ccb66-485cs + +* 查询 pod 暴露的 IP + Port + ```shell + $ kubectl describe pod bkm-operator-9964ccb66-485cs -n bkmonitor-operator + Name: bkm-operator-9964ccb66-485cs + Namespace: bkmonitor-operator + Priority: 0 + Service Account: bkmonitor-operator + Node: 127.0.7.101 + Labels: app.kubernetes.io/bk-component=bkmonitor-operator + pod-template-hash=9964ccb66 + Status: Running + IP: 127.1.11.11 + IPs: + IP: 127.1.11.11 + Controlled By: ReplicaSet/bkm-operator-9964ccb66 + Containers: + bkmonitor-operator: + Port: 8080/TCP + Host Port: 0/TCP + ``` + 可以看见 podMonitor 匹配到的 IP:Port 是 127.1.11.11:8080。这里的 IP 指的是 PodIP。 + +### GET /check/scrape + +> 检查采集任务抓取到的指标数量。 + +```shell +$ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/check/scrape | jq . +{ + "monitor_count": 16, + "lines_total": 557443, + "errors_total": 1, + "stats": [ + { + "monitor_name": "bkmonitor-operator-bkmonit-kubelet", + "namespace": "bkmonitor-operator", + "lines": 245004, + "errors": 0 + }, + { + "monitor_name": "bkmonitor-operator-bkmonit-kube-state-metrics", + "namespace": "bkmonitor-operator", + "lines": 140573, + "errors": 0 + }, + { + "monitor_name": "bkbase-pulsar-broker-http", + "namespace": "bkbase", + "lines": 85742, + "errors": 1 + } + ] +} +``` + +### GET /check/scrape/{namespace} + +> 抓取某个指定 namespace 指标文本并返回。 + +该接口会向指定的 namespace 下面的各个 serviceMonitor/podMonitor 的 Endpoints 抓取指标文本信息。 + +例如需要查询某一个具体的指标名对应的指标文本信息，这里我们通过 kube_endpoint_info 这个指标名进行过滤来展示。 + +```shell +# kube_endpoint_info 指标里面的 namespace 指采集目标的 namespace。 +$ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/check/scrape/bkmonitor-operator | grep kube_endpoint_info +kube_endpoint_info{namespace="bkapp-cc-portal-prod",endpoint="cc-portal--quality"} 1 +kube_endpoint_info{namespace="bkapp-cc-portal-prod",endpoint="cc-portal--auditlog"} 1 +kube_endpoint_info{namespace="bk-jaeger",endpoint="jaeger-collector"} 1 +kube_endpoint_info{namespace="bkapp-bkaidev-prod",endpoint="bkaidev-m-llm-gateway--web"} 1 +kube_endpoint_info{namespace="blueking",endpoint="job-config-watcher"} 1 +kube_endpoint_info{namespace="aiops-default",endpoint="service-9dc4ec690399daef49e0399d697dab67"} 1 +kube_endpoint_info{namespace="kube-system",endpoint="bkmonitor-operator-bkmonit-kube-proxy"} 1 +kube_endpoint_info{namespace="blueking",endpoint="bk-repo-bkrepo-repository"} 1 +kube_endpoint_info{namespace="bkbase",endpoint="bkbase-hive-primary-service-clusterip"} 1 +kube_endpoint_info{namespace="bkapp-bk0us0gsekit-prod",endpoint="bk0us0gsekit--pwatch"} 1 +kube_endpoint_info{namespace="aiops-default",endpoint="service-c22c7db09943e5c505af2b21129ce030"} 1 +kube_endpoint_info{namespace="bkmonitor-operator",endpoint="bkmonitor-operator-kubelet"} 1 +... +``` + +### GET /check/scrape/{namespace}/{monitor} + +> 抓取某个 namespace 下的 monitor 指标文本并返回。 + +同上，需要指定 serviceMonitor/podMonitor 名称信息。 + +### GET /check/active_discover + +> 检查活跃的 discover 情况。 + +discover 是各个 serviceMonitor/podMonitor 的 Spec 信息的监听器。一个 serviceMonitor/podMonitor 的 Endpoints Spec 可能配置了若干个采集端口，端口由 Index 来标识。 + +当用户需要检查自己配置的 serviceMonitor/podMonitor 抓取端口是否配置成功，可以使用这个接口来进行判断。若原本只配置了 2 个抓取端口，现在需要新增（或减少）一个，但是 Index 没有变化，说明未配置成功。 + +```shell +$ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/check/active_discover | jq . +[ + { + "name": "bkmonitor-operator-bkmonit-kubelet", + "kind": "ServiceMonitor", + "namespace": "bkmonitor-operator", + "index": 1 + }, + { + "name": "bkbase-dgraph-bkbase-dgr-zero", + "kind": "ServiceMonitor", + "namespace": "bkbase", + "index": 0 + }, + { + "name": "kg-dgraph-kg-dgraph-alpha", + "kind": "ServiceMonitor", + "namespace": "aiops-default", + "index": 0 + }, + { + "name": "bkmonitor-operator-bkmonit-apiserver", + "kind": "ServiceMonitor", + "namespace": "bkmonitor-operator", + "index": 0 + }, + { + "name": "kg-dgraph-kg-dgraph-zero", + "kind": "ServiceMonitor", + "namespace": "aiops-default", + "index": 0 + } +] +``` diff --git a/pkg/operator/docs/dataId.md b/pkg/operator/docs/dataId.md new file mode 100644 index 000000000..3457ee439 --- /dev/null +++ b/pkg/operator/docs/dataId.md @@ -0,0 +1,85 @@ +# 自定义资源 dataid 介绍 + +## 自定义资源 + +> 在介绍 `dataid` 之前，先来了解一下自定义资源 + +在 k8s 中除了内置资源 Pod、ReplicaSet、Deployment 等等之外，还支持自定义资源（CR），通常每一个自定义资源都需要有一个自定义资源定义（CRD）。有了 CRD 就可以自由地增加各种内置资源平级的资源。 + +先来看一下 k8s 中的资源 +```shell +$ kubectl api-resources -o wide +NAME SHORTNAMES APIVERSION NAMESPACED KIND VERBS CATEGORIES +bindings v1 true Binding create +componentstatuses cs v1 false ComponentStatus get,list +configmaps cm v1 true ConfigMap create,delete,deletecollection,get,list,patch,update,watch +endpoints ep v1 true Endpoints create,delete,deletecollection,get,list,patch,update,watch +events ev v1 true Event create,delete,deletecollection,get,list,patch,update,watch +limitranges limits v1 true LimitRange create,delete,deletecollection,get,list,patch,update,watch +namespaces ns v1 false Namespace create,delete,get,list,patch,update,watch +nodes no v1 false Node create,delete,deletecollection,get,list,patch,update,watch +persistentvolumeclaims pvc v1 true PersistentVolumeClaim create,delete,deletecollection,get,list,patch,update,watch +persistentvolumes pv v1 false PersistentVolume create,delete,deletecollection,get,list,patch,update,watch +pods po v1 true Pod create,delete,deletecollection,get,list,patch,update,watch all +podtemplates v1 true PodTemplate create,delete,deletecollection,get,list,patch,update,watch +replicationcontrollers rc v1 true ReplicationController create,delete,deletecollection,get,list,patch,update,watch all +resourcequotas quota v1 true ResourceQuota create,delete,deletecollection,get,list,patch,update,watch +secrets v1 true Secret create,delete,deletecollection,get,list,patch,update,watch +serviceaccounts sa v1 true ServiceAccount create,delete,deletecollection,get,list,patch,update,watch +services svc v1 true Service create,delete,get,list,patch,update,watch all +dataids bkd monitoring.bk.tencent.com/v1beta1 false DataID delete,deletecollection,get,list,patch,create,update,watch +... +``` + +可以看见 APIVERSION 为 `v1` 的均为 k8s 中的内置资源；而 dataids 是自定义资源，版本号为 `monitoring.bk.tencent.com/v1beta1` 。 + +## dataid 介绍 + +在蓝鲸监控中 dataid 用于对数据来源进行管理，这是管理端的特性，用户不感知。在容器环境中，dataid 是一种 CR。在用户集群接入蓝鲸监控之后，首先监控管理后端会对集群注入 3 个 dataid。分别是： +* custommetricdataid：自定义指标 dataid。 +* k8seventdataid：内置事件 dataid。 +* k8smetricdataid：内置指标 dataid。 + +```shell +$ kubectl get dataids -owide +NAME AGE +custommetricdataid 399d +k8seventdataid 399d +k8smetricdataid 399d +``` + +```shell +# 这里使用 `custommetricdataid` 这个 dataid 来举例子，具体看看 dataid 详细信息。 +$ kubectl get dataids custommetricdataid -oyaml +apiVersion: monitoring.bk.tencent.com/v1beta1 +kind: DataID +metadata: + creationTimestamp: "2023-07-19T10:40:03Z" + generation: 1 + labels: + isCommon: "true" + isSystem: "false" + usage: metric + name: custommetricdataid + resourceVersion: "5719372880" + selfLink: /apis/monitoring.bk.tencent.com/v1beta1/dataids/custommetricdataid + uid: 2f2f4b12-e63f-49d8-83e2-dd0d79f9fa16 +spec: + dataID: 1572865 + dimensionReplace: {} + labels: + bcs_cluster_id: BCS-K8S-00000 + bk_biz_id: "2" + metricReplace: {} +``` + +从上面的配置中，可以看见，`lebels` 中有： +- isCommon：是否为集群通用 dataid。 +- isSystem：是否为平台内置 dataid。 +- usage：数据类型标识，目前支持 metrics/event。 + +`Spec` 是该资源的详细规范 +- dataID：具体的 dataid 数值。 +- labels： + - bcs_cluster_id：bcs 集群 id。 + - bk_biz_id：bcs 集群关联的业务 id。 diff --git a/pkg/operator/docs/feature.md b/pkg/operator/docs/feature.md new file mode 100644 index 000000000..7f441c7d4 --- /dev/null +++ b/pkg/operator/docs/feature.md @@ -0,0 +1,241 @@ +## annotations features 配置规范 + +### keyForwardLocalhost + +用于检查采集端点是否需要重定向到 localhost。 + +例如： +将采集 IP 为 127.123.12.1 -> localhost + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + forwardLocalhost: "true" # 声明配置 +... +``` + +### normalizeMetricName + +用于检查是否需要标准化指标名。标准化指标名：将非数字字母转化为下划线，即将非 `[a-zA-Z0-9_]` 的字符全部替换成 `_` 。 + +此配置可以使得我们观察指标更加规范清晰。 + +例如： +```text +kube.deployment.replicas -> kube_deployment_replicas +kube:statefulset:replicas -> kube_statefulet_replicas +... +``` + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + normalizeMetricName: "true" # 声明配置 +... +``` + +### antiAffinity + +检查调度时是否需要反节点亲和。 + +反亲和性：对于有相同标签的 Pod，不希望把他们调度到相同的 node 下。 +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + antiAffinity: "true" # 声明配置 +... +``` + +### relabelRule + +通知采集器是否启用 workload、labeljoin 等特性。 + +* v1/workload：补充 pod workload 信息。 + ```shell + # 如 /workload/node/127.0.6.23 + $ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/workload/node/127.0.6.1 | jq . + + [ + { + "sourceLabels": [ + "namespace", + "pod_name" + ], + "separator": ";", + "regex": "kube-system;audit-prepare-127.0.6.1", + "targetLabel": "workload_kind", + "replacement": "Pod", + "action": "replace", + "nodeName": "127.0.6.1" + }, + { + "sourceLabels": [ + "namespace", + "pod_name" + ], + "separator": ";", + "regex": "kube-system;audit-prepare-127.0.6.1", + "targetLabel": "workload_name", + "replacement": "audit-prepare-127.0.6.1", + "action": "replace", + "nodeName": "127.0.6.1" + } + ] + ``` + 采集器根据 URL 返回的 relabels 配置进行 actions 处理，即补充上对应的 workload_name/workload_kind 信息。 + +* v2/workload：在 v1 版本的基础上，若存在 pod_name，查询参数添加上 pod_name。 + ```shell + # 如 /workload/node/worker1?podName=pod1 + $ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/workload/node/127.2.3.1?podName=pod1 + ``` + +* v1/labeljoin：**与 LabelJoinMatcher 搭配使用**，用于下发 kind、annotations、labels 等配置。 + ```shell + $ kubectl exec -it -n bkmonitor-operator bkm-operator-79486746f5-n6ztd -- curl http://localhost:8080/workload/node/127.2.3.1?annotations=annotations1&kind=Pod&labels=label1&podName=pod1&rules=labeljoin + ``` + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + relabelRule: "v1/workload,v2/workload,v1/node" # 声明配置 +... +``` + +### labelJoinMatcher + +在这里 `relabelRule: v1/labeljoin` 必须和 labelJoinMatcher 成对出现。 + +格式如下： +labelJoinMatcher: `{Kind}://annotation:{value1},annotation:{value2},label:{value3},label:{value4}` +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + relabelRule: "v1/labeljoin" # 声明配置 + labelJoinMatcher: "Pod://annotation:biz_service,label:deploy_zone" # 声明配置 +... +``` +使用上面的例子，生成的指标会带上 `annotation_` 或者 `label_` 前缀，并以匹配到的 label value 作为其值。如 `container_cpu_usage_seconds_total{annotation_biz_service="blueking",label_deploy_zone="gz"}`。 + +### relabelIndex + +指定的 serviceMonitor/porMonitor 中 Spec 中的 Endpoints 中的索引。Endpoints 是一个列表，relabelIndex 用于指定其中的一个端点。 + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + isSystem: "true" + meta.helm.sh/release-name: bkmonitor-operator + meta.helm.sh/release-namespace: bkmonitor-operator + relabelIndex: "1" + relabelRule: v1/workload + labels: + release: bkmonitor-operator + name: bkmonitor-operator-bkmonit-kubelet + namespace: bkmonitor-operator +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: https-metrics + relabelings: + - action: replace + sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + path: /metrics/cadvisor + port: https-metrics + relabelings: + - action: replace + sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + path: /metrics/probes + port: https-metrics + relabelings: + - action: replace + sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + jobLabel: k8s-app + namespaceSelector: + matchNames: + - bkmonitor-operator + selector: + matchLabels: + app.kubernetes.io/managed-by: bkmonitor-operator + k8s-app: kubelet +``` + +如下配置中 relabelIndex 为 0，就是具体指定上述 serviceMonitor Endpoints 列表中的第一个 Endpoint，即这部分。 + +```yaml + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: https-metrics + relabelings: + - action: replace + sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https +``` + +配置如下： + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + relabelIndex: "0" # 声明配置 +... +``` + +### monitorMatchSelector + +monitorMatchSelector 监控匹配的选择器，用于白名单过滤。 + +用来查看 targetGroup 中的 target 和 label 有没有能够匹配上白名单中的配置。 + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + monitorMatchSelector: "{key1}={value1},{key2}={value2}" # 声明配置 +... +``` + +### monitorDropSelector + +monitorDropSelector 监控跳过的选择器，用于黑名单过滤。 + +用于过滤掉 targetGroup 中的 target 和 label 被 BAN 的配置。 + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + monitorDropSelector: "{key1}={value1},{key2}={value2}" # 声明配置 +... +``` diff --git a/pkg/operator/docs/metrics.md b/pkg/operator/docs/metrics.md index 02aa911ae..14539a1d7 100644 --- a/pkg/operator/docs/metrics.md +++ b/pkg/operator/docs/metrics.md @@ -4,60 +4,44 @@ ### DataIdWatcher -| Metric | Type | Description | -| ------ |-------------------------------|----------------------------| -| dataid_info | Gauge | dataid 元数据信息 | -| dataid_watcher_handled_total | Counter | dataidwatcher 处理变更次数 | -| dataid_watcher_handled_duration_seconds | Histogram | dataidwatcher 处理变更耗时 | -| dataid_watcher_received_event_total | Counter | dataidwatcher 接收 k8s 事件计数器 | -| dataid_watcher_handled_event_total | Counter | dataidwatcher 处理 k8s 事件计数器 | +| Metric | Type | Description | +|------------------------------|---------|----------------------| +| dataid_info | Gauge | dataid 元数据信息 | +| dataid_watcher_handled_total | Counter | dataidwatcher 处理变更次数 | ### Discover -| Metric | Type | Description | -|----------------------------------------|-----------|--------------------| -| discover_started_total | Counter | discover 启动次数 | -| discover_stopped_total | Counter | discover 停止次数 | -| discover_waited_total | Counter | discover 等待停止次数 | -| discover_created_config_success_total | Counter | 创建子任务成功计数器 | -| discover_created_config_failed_total | Counter | 创建子任务失败计数器 | -| discover_removed_config_total | Counter | 移除子任务计数器 | -| discover_received_tg_total | Counter | 接收 targetgroup 计数器 | -| discover_handled_tg_duration_seconds | Histogram | 处理 targetgroup 耗时 | -| discover_accessed_secret_success_total | Counter | 获取 sercets 成功次数 | -| discover_accessed_secret_failed_total | Counter | 获取 sercets 失败次数 | - -### Workload - -| Metric | Type | Description | -| ------ |-----------|-------------| -| workload_lookup_request_total | Counter | 工作负载搜索次数 | -| workload_lookup_duration_seconds | Histogram | 工作负载搜索耗时 | +| Metric | Type | Description | +|---------------------------------------|---------|---------------------| +| discover_started_total | Counter | discover 启动次数 | +| discover_stopped_total | Counter | discover 停止次数 | +| discover_created_config_success_total | Counter | 创建子任务成功计数器 | +| discover_created_config_failed_total | Counter | 创建子任务失败计数器 | +| discover_created_config_cached_total | Counter | 创建子任务缓存计数器 | +| discover_handled_tg_total | Counter | 已处理 targetgroup 计数器 | +| discover_deleted_tg_source_total | Counter | 已删除 targetgroup 计数器 | +| monitor_scrape_interval_seconds | Gauge | 监控采集时间间隔 | ### Operator -| Metric | Type | Description | -|-----------------------|-----------------------|-----------------------------| -| uptime | Counter | 进程运行时间 | -| cluster_version | Gauge | kubernetes 服务端版本 | -| build_info | Gauge | 构建版本信息 | -| active_config_count | Gauge | 活跃的采集任务数量 | -| active_shared_discovery_count | Gauge | 活跃的 shared_discovery 数量 | -| active_monitor_resource_count | Gauge | 活跃的监控资源数量 | -| received_event_total | Counter | 接收 k8s 接收事件总数 | -| handled_event_total | Counter | 处理 k8s 接收事件总数 | -| handled_event_duration_seconds | Histogram | 处理 k8s 事件耗时 | -| handled_secret_success_total | Counter | 处理 secrets 成功次数 | -| handled_secret_failed_total | Counter | 处理 secrets 失败次数 | -| dispatched_task_total | Counter | 分派任务计数器 | -| dispatched_task_duration_seconds | Histogram | 分派任务耗时 | -| skipped_secret_total | Counter | secrets 内容无变更 直接跳过计数 | -| compressed_config_failed_total | Counter | 压缩配置文件错误计数器 | -| handled_discover_notify_total | Counter | 处理 discover notify 计数器 | -| handled_dataid_watcher_notify_total | Counter | 处理 dataidwatcher notify 计数器 | -| reloaded_discover_duration_seconds | Histogram | 重载 discover 耗时 | -| active_secret_file_count | Gauge | 活跃的 sercets 数量 | -| active_secret_bytes | Gauge | 活跃的 sercets 字节大小 | -| secrets_exceeded | Counter | sercets 超限计数器 | -| scaled_statefulset_failed_total | Counter | 弹缩 statefulset worker 失败次数 | -| scaled_statefulset_success_total | Counter | 弹缩 statefulset worker 成功次数 | +| Metric | Type | Description | +|----------------------------------|-----------|---------------------------------------------| +| uptime | Counter | 进程运行时间 | +| cluster_version | Gauge | kubernetes 服务端版本 | +| build_info | Gauge | 构建版本信息 | +| handled_event_total | Counter | 处理 k8s 接收事件总数 | +| handled_secret_success_total | Counter | 处理 secrets 成功次数 | +| handled_secret_failed_total | Counter | 处理 secrets 失败次数 | +| dispatched_task_total | Counter | 分派任务计数器 | +| dispatched_task_duration_seconds | Histogram | 分派任务耗时 | +| active_secret_file_count | Gauge | 活跃的 sercets 数量 | +| secrets_exceeded | Counter | sercets 超限计数器 | +| scaled_statefulset_failed_total | Counter | 弹缩 statefulset worker 失败次数 | +| scaled_statefulset_success_total | Counter | 弹缩 statefulset worker 成功次数 | +| node_config_count | Gauge | node 上采集任务数量 | +| monitor_endpoint_count | Gauge | serviceMonitor/podMonitor 匹配到的 endpoints 数量 | +| workload_count | Gauge | 工作负载数量 | +| node_count | Gauge | 节点数量 | +| shared_discovery_count | Gauge | shared_discovery 数量 | +| discover_count | Gauge | discover 数量 | +| statefulset_workers | Gauge | statefulset_workers 数量 | diff --git a/pkg/operator/go.mod b/pkg/operator/go.mod index 3a81bb030..6ca79145d 100644 --- a/pkg/operator/go.mod +++ b/pkg/operator/go.mod @@ -31,6 +31,7 @@ require ( github.com/Tencent/bk-bcs/bcs-scenarios/kourse v0.0.0-20220914032224-06b1bd3358bc github.com/TencentBlueKing/bkmonitor-datalink/pkg/libgse v0.0.0-00010101000000-000000000000 github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils v0.0.0-00010101000000-000000000000 + github.com/blang/semver/v4 v4.0.0 github.com/cespare/xxhash/v2 v2.2.0 github.com/ghodss/yaml v1.0.0 github.com/valyala/bytebufferpool v1.0.0 diff --git a/pkg/operator/go.sum b/pkg/operator/go.sum index e863de70f..4de4a3128 100644 --- a/pkg/operator/go.sum +++ b/pkg/operator/go.sum @@ -161,6 +161,7 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= diff --git a/pkg/operator/operator/discover/discover.go b/pkg/operator/operator/discover/discover.go index 6ae54c7e8..0a95872fd 100644 --- a/pkg/operator/operator/discover/discover.go +++ b/pkg/operator/operator/discover/discover.go @@ -37,6 +37,7 @@ import ( bkv1beta1 "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/apis/crd/v1beta1" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/define" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/eplabels" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/feature" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/k8sutils" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/labelspool" @@ -108,7 +109,7 @@ type ChildConfig struct { } func (c ChildConfig) String() string { - return fmt.Sprintf("Node=%s, FileName=%s, Address=%s, Data=%s", c.Node, c.FileName, c.Address, string(c.Data)) + return fmt.Sprintf("Node=%s, FileName=%s, Address=%s", c.Node, c.FileName, c.Address) } func (c ChildConfig) Hash() uint64 { @@ -152,6 +153,7 @@ type BaseParams struct { MatchSelector map[string]string DropSelector map[string]string LabelJoinMatcher *feature.LabelJoinMatcherSpec + UseEndpointSlice bool } type BaseDiscover struct { @@ -273,15 +275,15 @@ func (d *BaseDiscover) makeMetricTarget(lbls, origLabels labels.Labels, namespac for _, label := range origLabels { switch label.Name { // 补充 NodeName - case labelEndpointNodeName, labelPodNodeName: + case eplabels.EndpointNodeName(d.UseEndpointSlice), labelPodNodeName: metricTarget.NodeName = label.Value // 如果 target 类型是 node，则需要特殊处理，此时 endpointNodeName 对应 label 会为空 - case labelEndpointAddressTargetKind, labelPodAddressTargetKind: + case eplabels.EndpointAddressTargetKind(d.UseEndpointSlice), labelPodAddressTargetKind: if label.Value == "Node" { isNodeType = true } - case labelEndpointAddressTargetName, labelPodAddressTargetName: + case eplabels.EndpointAddressTargetName(d.UseEndpointSlice), labelPodAddressTargetName: targetName = label.Value } } diff --git a/pkg/operator/operator/discover/endpoint.go b/pkg/operator/operator/discover/endpoint.go index 47b60a31c..4df1f4a0d 100644 --- a/pkg/operator/operator/discover/endpoint.go +++ b/pkg/operator/operator/discover/endpoint.go @@ -16,16 +16,18 @@ import ( ) const ( - labelEndpointNodeName = "__meta_kubernetes_endpoint_node_name" - labelPodNodeName = "__meta_kubernetes_pod_node_name" - labelEndpointAddressTargetKind = "__meta_kubernetes_endpoint_address_target_kind" - labelPodAddressTargetKind = "__meta_kubernetes_pod_address_target_kind" - labelEndpointAddressTargetName = "__meta_kubernetes_endpoint_address_target_name" - labelPodAddressTargetName = "__meta_kubernetes_pod_address_target_name" - - discoverTypeEndpoints = "endpoints" + labelPodNodeName = "__meta_kubernetes_pod_node_name" + labelPodAddressTargetKind = "__meta_kubernetes_pod_address_target_kind" + labelPodAddressTargetName = "__meta_kubernetes_pod_address_target_name" ) +func discoverTypeEndpoints(endpointslice bool) string { + if endpointslice { + return "endpointslice" + } + return "endpoints" +} + type EndpointParams struct { *BaseParams } @@ -36,12 +38,12 @@ type Endpoint struct { func NewEndpointDiscover(ctx context.Context, meta define.MonitorMeta, checkFn define.CheckFunc, params *EndpointParams) Discover { return &Endpoint{ - BaseDiscover: NewBaseDiscover(ctx, discoverTypeEndpoints, meta, checkFn, params.BaseParams), + BaseDiscover: NewBaseDiscover(ctx, discoverTypeEndpoints(params.UseEndpointSlice), meta, checkFn, params.BaseParams), } } func (d *Endpoint) Type() string { - return discoverTypeEndpoints + return discoverTypeEndpoints(d.UseEndpointSlice) } func (d *Endpoint) Reload() error { @@ -51,7 +53,7 @@ func (d *Endpoint) Reload() error { func (d *Endpoint) Start() error { d.PreStart() - RegisterSharedDiscover(discoverTypeEndpoints, d.KubeConfig, d.getNamespaces()) + RegisterSharedDiscover(discoverTypeEndpoints(d.UseEndpointSlice), d.KubeConfig, d.getNamespaces()) d.wg.Add(1) go func() { diff --git a/pkg/operator/operator/discover/hook.go b/pkg/operator/operator/discover/hook.go index 868391e4a..dde9eca9a 100644 --- a/pkg/operator/operator/discover/hook.go +++ b/pkg/operator/operator/discover/hook.go @@ -20,9 +20,7 @@ const ( confDefaultPeriodPath = "discover.scrape.default_period" ) -var ( - ConfDefaultPeriod string -) +var ConfDefaultPeriod string func initConfig() { viper.SetDefault(confDefaultPeriodPath, "60s") diff --git a/pkg/operator/operator/hook.go b/pkg/operator/operator/hook.go index 389d8581f..58b189f90 100644 --- a/pkg/operator/operator/hook.go +++ b/pkg/operator/operator/hook.go @@ -16,7 +16,6 @@ import ( "github.com/spf13/viper" "k8s.io/client-go/rest" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/define" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/config" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/operator/target" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" @@ -40,6 +39,7 @@ const ( confEnablePromRulePath = "operator.enable_prometheus_rule" confEnableStatefulSetWorkerPath = "operator.enable_statefulset_worker" confEnableDaemonSetWorkerPath = "operator.enable_daemonset_worker" + confEnableEndpointslicePath = "operator.enable_endpointslice" confKubeletNamespacePath = "operator.kubelet.namespace" confKubeletNamePath = "operator.kubelet.name" confKubeletEnablePath = "operator.kubelet.enable" @@ -97,11 +97,11 @@ var ( ConfEnablePromRule bool ConfEnableStatefulSetWorker bool ConfEnableDaemonSetWorker bool + ConfEnableEndpointslice bool ConfKubeletNamespace string ConfKubeletName string ConfKubeletEnable bool ConfMaxNodeSecretRatio float64 - ConfPodName string ConfStatefulSetWorkerHpa bool ConfStatefulSetWorkerFactor int ConfStatefulSetReplicas int @@ -149,7 +149,6 @@ func initConfig() { viper.SetDefault(confMonitorNamespacePath, "bkmonitor-operator") viper.SetDefault(confEnableServiceMonitorPath, true) viper.SetDefault(confEnablePodMonitorPath, true) - viper.SetDefault(confEnablePromRulePath, false) viper.SetDefault(confEnableStatefulSetWorkerPath, true) viper.SetDefault(confEnableDaemonSetWorkerPath, true) viper.SetDefault(confKubeletNamePath, "bkmonitor-operator-kubelet") @@ -181,6 +180,7 @@ func updateConfig() { ConfEnablePromRule = viper.GetBool(confEnablePromRulePath) ConfEnableStatefulSetWorker = viper.GetBool(confEnableStatefulSetWorkerPath) ConfEnableDaemonSetWorker = viper.GetBool(confEnableDaemonSetWorkerPath) + ConfEnableEndpointslice = viper.GetBool(confEnableEndpointslicePath) ConfKubeletNamespace = viper.GetString(confKubeletNamespacePath) ConfKubeletName = viper.GetString(confKubeletNamePath) ConfKubeletEnable = viper.GetBool(confKubeletEnablePath) @@ -191,7 +191,6 @@ func updateConfig() { KeyFile: viper.GetString(confTLSKeyFilePath), CAFile: viper.GetString(confTLSCAFilePath), } - ConfPodName = viper.GetString(define.EnvPodName) ConfStatefulSetWorkerHpa = viper.GetBool(confStatefulSetWorkerHpaPath) ConfStatefulSetWorkerFactor = viper.GetInt(confStatefulSetWorkerFactorPath) ConfStatefulSetReplicas = viper.GetInt(confStatefulSetReplicasPath) diff --git a/pkg/operator/operator/metrics.go b/pkg/operator/operator/metrics.go index 8a2afabbd..6182143f1 100644 --- a/pkg/operator/operator/metrics.go +++ b/pkg/operator/operator/metrics.go @@ -19,6 +19,15 @@ import ( ) var ( + clusterVersion = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: define.MonitorNamespace, + Name: "cluster_version", + Help: "kubernetes server version", + }, + []string{"version"}, + ) + appUptime = promauto.NewCounter( prometheus.CounterOpts{ Namespace: define.MonitorNamespace, @@ -240,3 +249,7 @@ func (m *metricMonitor) IncScaledStatefulSetSuccessCounter() { func (m *metricMonitor) SetStatefulSetWorkerCount(count int) { statefulSetWorkerCount.Set(float64(count)) } + +func (m *metricMonitor) SetKubernetesVersion(v string) { + clusterVersion.WithLabelValues(v).Set(1) +} diff --git a/pkg/operator/operator/objectsref/controller.go b/pkg/operator/operator/objectsref/controller.go index 41ce5a648..34d6759c6 100644 --- a/pkg/operator/operator/objectsref/controller.go +++ b/pkg/operator/operator/objectsref/controller.go @@ -223,13 +223,7 @@ func NewController(ctx context.Context, client kubernetes.Interface, tkexClient cancel: cancel, } - version, err := client.Discovery().ServerVersion() - if err != nil { - return nil, err - } - KubernetesServerVersion = version.String() - setClusterVersion(KubernetesServerVersion) - + var err error resources := listServerPreferredResources(client.Discovery()) sharedInformer := informers.NewSharedInformerFactoryWithOptions(client, define.ReSyncPeriod, informers.WithNamespace(metav1.NamespaceAll)) diff --git a/pkg/operator/operator/objectsref/extend.go b/pkg/operator/operator/objectsref/extend.go index aa07f7ab9..90d720726 100644 --- a/pkg/operator/operator/objectsref/extend.go +++ b/pkg/operator/operator/objectsref/extend.go @@ -68,8 +68,6 @@ var ( Resource: resourceGameDeployments, Kind: kindGameDeployment, } - - KubernetesServerVersion string ) type tkexObjects struct { diff --git a/pkg/operator/operator/objectsref/metrics.go b/pkg/operator/operator/objectsref/metrics.go index 233028a42..32d07a197 100644 --- a/pkg/operator/operator/objectsref/metrics.go +++ b/pkg/operator/operator/objectsref/metrics.go @@ -12,20 +12,6 @@ package objectsref import ( "sync" "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/define" -) - -var clusterVersion = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: define.MonitorNamespace, - Name: "cluster_version", - Help: "kubernetes server version", - }, - []string{"version"}, ) type namespaceKind struct { @@ -80,7 +66,3 @@ func decClusterNodeCount() { clusterNode-- clusterNodeUpdatedAt = time.Now() } - -func setClusterVersion(v string) { - clusterVersion.WithLabelValues(v).Set(1) -} diff --git a/pkg/operator/operator/operator.go b/pkg/operator/operator/operator.go index 72d07056f..6596c89b3 100644 --- a/pkg/operator/operator/operator.go +++ b/pkg/operator/operator/operator.go @@ -17,6 +17,7 @@ import ( "time" tkexversiond "github.com/Tencent/bk-bcs/bcs-scenarios/kourse/pkg/client/clientset/versioned" + "github.com/blang/semver/v4" "github.com/pkg/errors" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promversioned "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" @@ -44,6 +45,11 @@ const ( monitorKindPodMonitor = "PodMonitor" ) +var ( + kubernetesVersion string + useEndpointslice bool +) + // Operator 负责部署和调度任务 type Operator struct { ctx context.Context @@ -129,6 +135,25 @@ func NewOperator(ctx context.Context, buildInfo BuildInfo) (*Operator, error) { denyTargetNamespaces[namespace] = struct{}{} } + version, err := operator.client.Discovery().ServerVersion() + if err != nil { + return nil, err + } + kubernetesVersion = version.String() + operator.mm.SetKubernetesVersion(kubernetesVersion) + + parsedVersion, err := semver.ParseTolerant(kubernetesVersion) + if err != nil { + parsedVersion = semver.MustParse("1.12.0") // 最低支持的 k8s 版本 + logger.Errorf("parse kubernetes version failed, instead of '%v', err: %v", parsedVersion, err) + } + + // 1.21.0 开始 endpointslice 正式成为 v1 + useEndpointslice = parsedVersion.GTE(semver.MustParse("1.21.0")) && ConfEnableEndpointslice + if useEndpointslice { + logger.Info("use 'endpointslice' instead of 'endpoint'") + } + if ConfEnableServiceMonitor { operator.serviceMonitorInformer, err = prominformers.NewInformersForResource( prominformers.NewMonitoringInformerFactories( @@ -179,7 +204,7 @@ func NewOperator(ctx context.Context, buildInfo BuildInfo) (*Operator, error) { if err != nil { return nil, errors.Wrap(err, "create PrometheusRule informer failed") } - operator.promsliController = promsli.NewController(operator.ctx, operator.client) + operator.promsliController = promsli.NewController(operator.ctx, operator.client, useEndpointslice) } operator.objectsController, err = objectsref.NewController(operator.ctx, operator.client, operator.tkexclient) @@ -567,6 +592,7 @@ func (c *Operator) createServiceMonitorDiscovers(serviceMonitor *promv1.ServiceM MatchSelector: feature.MonitorMatchSelector(serviceMonitor.Annotations), DropSelector: feature.MonitorDropSelector(serviceMonitor.Annotations), LabelJoinMatcher: feature.LabelJoinMatcher(serviceMonitor.Annotations), + UseEndpointSlice: useEndpointslice, Name: monitorMeta.ID(), DataID: dataID, KubeConfig: ConfKubeConfig, @@ -638,7 +664,7 @@ func (c *Operator) handleServiceMonitorUpdate(oldObj interface{}, newObj interfa } if old.ResourceVersion == cur.ResourceVersion { - logger.Debugf("serviceMonitor %+v does not change", old) + logger.Debugf("serviceMonitor '%s/%s' does not change", old.Namespace, old.Name) return } @@ -768,6 +794,7 @@ func (c *Operator) createPodMonitorDiscovers(podMonitor *promv1.PodMonitor) []di MatchSelector: feature.MonitorMatchSelector(podMonitor.Annotations), DropSelector: feature.MonitorDropSelector(podMonitor.Annotations), LabelJoinMatcher: feature.LabelJoinMatcher(podMonitor.Annotations), + UseEndpointSlice: useEndpointslice, Name: monitorMeta.ID(), DataID: dataID, KubeConfig: ConfKubeConfig, @@ -861,7 +888,7 @@ func (c *Operator) handlePodMonitorUpdate(oldObj interface{}, newObj interface{} } if old.ResourceVersion == cur.ResourceVersion { - logger.Debugf("podMonitor %+v does not change", old) + logger.Debugf("podMonitor '%s/%s' does not change", old.Namespace, old.Name) return } diff --git a/pkg/operator/operator/promsli/promsli.go b/pkg/operator/operator/promsli/promsli.go index faa12af7a..5316055c9 100644 --- a/pkg/operator/operator/promsli/promsli.go +++ b/pkg/operator/operator/promsli/promsli.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/compressor" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/eplabels" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/feature" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/k8sutils" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/notifier" @@ -50,29 +51,31 @@ type Controller struct { client kubernetes.Interface bus *notifier.RateBus - mut sync.Mutex - rules map[string]*promv1.PrometheusRule - rulesRelation map[string]string - smMetrics map[string]map[string]struct{} - serviceMonitors map[string]*promv1.ServiceMonitor - registerRules map[string]struct{} + mut sync.Mutex + rules map[string]*promv1.PrometheusRule + rulesRelation map[string]string + smMetrics map[string]map[string]struct{} + serviceMonitors map[string]*promv1.ServiceMonitor + registerRules map[string]struct{} + useEndpointslice bool prevScrapeContent []byte prevRuleContent map[string]string } -func NewController(ctx context.Context, client kubernetes.Interface) *Controller { +func NewController(ctx context.Context, client kubernetes.Interface, useEndpointslice bool) *Controller { ctx, cancel := context.WithCancel(ctx) c := &Controller{ - ctx: ctx, - cancel: cancel, - client: client, - bus: notifier.NewDefaultRateBus(), - rules: make(map[string]*promv1.PrometheusRule), - rulesRelation: make(map[string]string), - smMetrics: map[string]map[string]struct{}{}, - serviceMonitors: make(map[string]*promv1.ServiceMonitor), - registerRules: map[string]struct{}{}, + ctx: ctx, + cancel: cancel, + client: client, + bus: notifier.NewDefaultRateBus(), + rules: make(map[string]*promv1.PrometheusRule), + rulesRelation: make(map[string]string), + smMetrics: map[string]map[string]struct{}{}, + useEndpointslice: useEndpointslice, + serviceMonitors: make(map[string]*promv1.ServiceMonitor), + registerRules: map[string]struct{}{}, } go c.handle() @@ -333,7 +336,7 @@ func (c *Controller) generateServiceMonitorScrapeConfigs() []yaml.MapSlice { // 内置白名单 if feature.SliMonitor(sm.Annotations) == sliAnnotationBuiltin { for i, ep := range sm.Spec.Endpoints { - cfg = append(cfg, generateServiceMonitorScrapeConfig(sm, ep, i, nil)) + cfg = append(cfg, c.generateServiceMonitorScrapeConfig(sm, ep, i, nil)) } continue } @@ -360,13 +363,13 @@ func (c *Controller) generateServiceMonitorScrapeConfigs() []yaml.MapSlice { } for i, ep := range sm.Spec.Endpoints { - cfg = append(cfg, generateServiceMonitorScrapeConfig(sm, ep, i, keep)) + cfg = append(cfg, c.generateServiceMonitorScrapeConfig(sm, ep, i, keep)) } } return cfg } -func generateServiceMonitorScrapeConfig(sm *promv1.ServiceMonitor, ep promv1.Endpoint, index int, keepMetrics []string) yaml.MapSlice { +func (c *Controller) generateServiceMonitorScrapeConfig(sm *promv1.ServiceMonitor, ep promv1.Endpoint, index int, keepMetrics []string) yaml.MapSlice { cfg := yaml.MapSlice{ { Key: "job_name", @@ -447,10 +450,9 @@ func generateServiceMonitorScrapeConfig(sm *promv1.ServiceMonitor, ep promv1.End // Filter targets based on correct port for the endpoint. if ep.Port != "" { - sourceLabels := []string{"__meta_kubernetes_endpoint_port_name"} relabelings = append(relabelings, yaml.MapSlice{ {Key: "action", Value: "keep"}, - yaml.MapItem{Key: "source_labels", Value: sourceLabels}, + yaml.MapItem{Key: "source_labels", Value: []string{eplabels.EndpointPortName(c.useEndpointslice)}}, {Key: "regex", Value: ep.Port}, }) } else if ep.TargetPort != nil { @@ -469,7 +471,7 @@ func generateServiceMonitorScrapeConfig(sm *promv1.ServiceMonitor, ep promv1.End } } - sourceLabels := []string{"__meta_kubernetes_endpoint_address_target_kind", "__meta_kubernetes_endpoint_address_target_name"} + sourceLabels := []string{eplabels.EndpointAddressTargetKind(c.useEndpointslice), eplabels.EndpointAddressTargetName(c.useEndpointslice)} // Relabel namespace and pod and service labels into proper labels. relabelings = append(relabelings, []yaml.MapSlice{ { // Relabel node labels with meta labels available with Prometheus >= v2.3. diff --git a/pkg/operator/operator/relabel.go b/pkg/operator/operator/relabel.go index 75ed95fd5..937464d2e 100644 --- a/pkg/operator/operator/relabel.go +++ b/pkg/operator/operator/relabel.go @@ -19,6 +19,8 @@ import ( "github.com/prometheus/prometheus/model/relabel" "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/common/eplabels" ) func sanitizeLabelName(name string) string { @@ -149,7 +151,7 @@ func getServiceMonitorRelabels(m *promv1.ServiceMonitor, ep *promv1.Endpoint) [] if ep.Port != "" { relabelings = append(relabelings, yaml.MapSlice{ {Key: "action", Value: "keep"}, - {Key: "source_labels", Value: []string{"__meta_kubernetes_endpoint_port_name"}}, + {Key: "source_labels", Value: []string{eplabels.EndpointPortName(useEndpointslice)}}, {Key: "regex", Value: ep.Port}, }) } else if ep.TargetPort != nil { @@ -171,14 +173,14 @@ func getServiceMonitorRelabels(m *promv1.ServiceMonitor, ep *promv1.Endpoint) [] // Relabel namespace and pod and service labels into proper labels. relabelings = append(relabelings, []yaml.MapSlice{ { // Relabel node labels for pre v2.3 meta labels - {Key: "source_labels", Value: []string{"__meta_kubernetes_endpoint_address_target_kind", "__meta_kubernetes_endpoint_address_target_name"}}, + {Key: "source_labels", Value: []string{eplabels.EndpointAddressTargetKind(useEndpointslice), eplabels.EndpointAddressTargetName(useEndpointslice)}}, {Key: "separator", Value: ";"}, {Key: "regex", Value: "Node;(.*)"}, {Key: "replacement", Value: "${1}"}, {Key: "target_label", Value: "node"}, }, { // Relabel pod labels for >=v2.3 meta labels - {Key: "source_labels", Value: []string{"__meta_kubernetes_endpoint_address_target_kind", "__meta_kubernetes_endpoint_address_target_name"}}, + {Key: "source_labels", Value: []string{eplabels.EndpointAddressTargetKind(useEndpointslice), eplabels.EndpointAddressTargetName(useEndpointslice)}}, {Key: "separator", Value: ";"}, {Key: "regex", Value: "Pod;(.*)"}, {Key: "replacement", Value: "${1}"}, diff --git a/pkg/operator/operator/server.go b/pkg/operator/operator/server.go index b6cb74ba1..f19f0f9c3 100644 --- a/pkg/operator/operator/server.go +++ b/pkg/operator/operator/server.go @@ -171,7 +171,7 @@ const ( formatCheckDataIDFailedMsg = ` [x] check dataids - Description: 期待 dataids 数量应大于等于 3 个，目前发现 %d 个 -- Suggestion: dataid 是由 metadata 组件注入，请确定接入流程是否规范。同时检查 metadata 日志，确定是否出现异常（必要时携带日志联系开发或者运维同学） +- Suggestion: dataid 由 metadata 组件注入，请确定接入流程是否规范。同时检查 metadata 日志，确定是否出现异常。 * operator 从启动到监听 dataids 资源可能存在约 20s 的延迟 * 监控后台为传统部署，日志路径为 /data/bkee/logs/bkmonitorv3/kernel_metadata.log * 监控后台为容器部署，请查看 bkmonitor-alarm-cron-worker pod 的日志 @@ -189,7 +189,7 @@ const ( [√] check namespaces - Description: 监测 namespace 白名单列表 %v，namespace 黑名单列表 %v - Suggestion: 请检查所需监控资源是否位于监测命名空间列表下，黑名单只在白名单列表为空时生效 - * 如若发现所需命名空间没有在监测列表中，请修改 values.yaml 中的 denyTargetNamespaces 或者 targetNamespaces，并 'helm upgrade' 到集群中 + * 如若发现所需命名空间没有在监测列表中，请修改 values.yaml 中的 denyTargetNamespaces 或者 targetNamespaces，并更新到集群中 ` formatCheckNamespaceFailedMsg = ` [x] check namespaces @@ -214,7 +214,8 @@ const ( formatScrapeMsg = ` [√] check scrape stats - Description: 总共发现 %d 个 monitor 资源，抓取数据行数为 %d，采集共出现 %d 次错误，更新时间 %s -- Suggestion: 错误可能由 forwardLocal 导致（可忽略），可过滤 'scrape error' 关键字查看详细错误信息。部分指标会有黑白名单机制，此抓取数据不做任何过滤 +- Suggestion: 错误可能由 forwardLocal 导致（可忽略），可过滤 'scrape error' 关键字查看详细错误信息。 +* 部分指标会有黑白名单机制，此抓取数据不做任何过滤。 * TOP%d 数据量如下，详细情况可访问 /check/scrape 路由。%s %s ` @@ -245,19 +246,17 @@ const ( ` formatOperatorLogMsg = ` [o] bkmonitor-operator logs -- Description: 使用 'kubectl logs -n ${.Release.Namespace} ${bkm-operator-pod}' 查看是否有 ERROR 信息 -- Suggestion: 检查 ERROR 日志是否有明显的报错信息，必要时携带日志联系开发或者运维同学 +- Description: 使用 'kubectl logs -n ${.Namespace} ${bkm-operator-pod}' 查看是否有 ERROR 信息。 ` formatBkmonitorbeatTroubleshootingMsg = ` [o] bkmonitorbeat troubleshooting - Description: 如若上述检查无发现异常问题，则考虑排查 bkmonitorbeat 本身的采集是否出现异常 -- Suggestion: 使用 strace 命令抓取 bkmonitorbeat write syscall 数据 - 1）根据上述检查得到采集任务所在节点，并使用 'kubectl get pods -n ${.Release.Namespace} -owide' 确定对应的 bkmonitorbeat pod - 2）使用 'kubectl logs -n ${.Release.Namespace} ${bkm-operator-pod}' 查看是否有 ERROR 信息 - 3）使用 'kubectl exec it -n ${.Release.Namespace} ${bkmonitorbeat-pod}' 命令查看 bkmonitorbeat 所在进程 pid - 4）使用 'kubectl exec' 执行 'strace -p ${pid} -s 1024000 -f -e write 2>&1 > /tmp/bkmonitorbeat.strace' 等待一分钟导出 strace 数据 - 5）过滤 *.strace 文件查看是否有采集任务指标对应的关键字，判断数据是否有写入到 gse sockets，如若有写到 gse 则说明 bkmonitorbeat 本身没问题，需要排查链路问题 - 6）链路排查可按照二进制部署排查思路 kafka -> transfer -> influxdb-proxy -> influxdb（必要时携带日志联系开发或者运维同学） +- Suggestion: 优先检查采集器日志是否有异常，采集器会在每次采集记录日志流水 + 1）根据上述检查得到采集任务所在节点，并使用 'kubectl get pods -n ${.Namespace} -owide' 确定对应的采集器 pod + 2）使用 'kubectl exec it -n ${.Namespace} ${worker-pod}' 命令查看 bkmonitorbeat 所在进程 pid + 3）使用 'kubectl exec' 执行 'strace -p ${pid} -s 1024000 -f -e write 2>&1 > /tmp/strace' 等待一分钟导出 strace 数据 + 4）过滤 *.strace 文件查看是否有采集任务指标对应的关键字，判断数据是否有写入到 gse sockets，如若有写到 gse 则说采集工作正常，需要排查链路问题 + 5）链路排查可按照二进制部署排查思路 kafka -> transfer -> influxdb-proxy -> influxdb ` ) @@ -280,10 +279,10 @@ func (c *Operator) CheckRoute(w http.ResponseWriter, r *http.Request) { var b []byte // 检查 kubernetes 版本信息 - if objectsref.KubernetesServerVersion == "" { + if kubernetesVersion == "" { buf.WriteString(formatKubernetesVersionFailedMsg) } else { - buf.WriteString(fmt.Sprintf(formatKubernetesVersionSuccessMsg, objectsref.KubernetesServerVersion)) + buf.WriteString(fmt.Sprintf(formatKubernetesVersionSuccessMsg, kubernetesVersion)) } // 检查 bkmonitor-operator 版本信息 diff --git a/pkg/operator/operator/statefulset.go b/pkg/operator/operator/statefulset.go index 7f2ab248c..aa73e7604 100644 --- a/pkg/operator/operator/statefulset.go +++ b/pkg/operator/operator/statefulset.go @@ -104,7 +104,7 @@ func (c *Operator) handleStatefulSetWorkerUpdate(oldObj, newObj interface{}) { } if old.ResourceVersion == cur.ResourceVersion { - logger.Debugf("StatefulSet %+v does not change", old) + logger.Debugf("StatefulSet '%s/%s' does not change", old.Namespace, old.Name) return } diff --git a/pkg/operator/operator/target/event_test.go b/pkg/operator/operator/target/event_test.go index 02f8e4fa0..b17d9edd7 100644 --- a/pkg/operator/operator/target/event_test.go +++ b/pkg/operator/operator/target/event_test.go @@ -33,6 +33,7 @@ name: event_collect version: "1" task_id: 1 dataid: 123 +upmetrics_dataid: 0 interval: 1m event_span: 2h tail_files: diff --git a/pkg/unify-query/bkapi/bkapi.go b/pkg/unify-query/bkapi/bkapi.go new file mode 100644 index 000000000..943107474 --- /dev/null +++ b/pkg/unify-query/bkapi/bkapi.go @@ -0,0 +1,65 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package bkapi + +import ( + "fmt" + "sync" + + "github.com/spf13/viper" +) + +const ( + AdminUserName = "admin" + BkApiAuthorization = "X-Bkapi-Authorization" +) + +type BkApi struct { + address string + + code string + secret string +} + +var ( + onceBkApi sync.Once + defaultBkApi *BkApi +) + +func GetBkApi() *BkApi { + onceBkApi.Do(func() { + defaultBkApi = &BkApi{ + address: viper.GetString(BkApiAddressConfigPath), + code: viper.GetString(BkApiCodeConfigPath), + secret: viper.GetString(BkApiSecretConfigPath), + } + }) + + return defaultBkApi +} + +func (i *BkApi) Headers(headers map[string]string) map[string]string { + if len(headers) == 0 { + headers = make(map[string]string) + } + headers[BkApiAuthorization] = fmt.Sprintf( + `{"bk_username": "%s", "bk_app_code": "%s", "bk_app_secret": "%s"}`, + AdminUserName, i.code, i.secret, + ) + return headers +} + +func (i *BkApi) Url(path string) string { + url := i.address + if path != "" { + url = fmt.Sprintf("%s/%s", i.address, path) + } + return url +} diff --git a/pkg/unify-query/bkapi/bkdata.go b/pkg/unify-query/bkapi/bkdata.go new file mode 100644 index 000000000..3f232f62b --- /dev/null +++ b/pkg/unify-query/bkapi/bkdata.go @@ -0,0 +1,90 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package bkapi + +import ( + "fmt" + "net/http" + "sync" + + "github.com/spf13/viper" +) + +const ( + BkDataAuthorization = "X-Bkbase-Authorization" + + QuerySync = "query_sync" + QueryAsync = "query_async" +) + +var ( + onceBkDataApi sync.Once + defaultBkDataApi *BkDataApi +) + +type BkDataApi struct { + bkApi *BkApi + + uriPath string + token string + authenticationMethod string +} + +func GetBkDataApi() *BkDataApi { + onceBkDataApi.Do(func() { + defaultBkDataApi = &BkDataApi{ + bkApi: GetBkApi(), + token: viper.GetString(BkDataTokenConfigPath), + authenticationMethod: viper.GetString(BkDataAuthenticationMethodConfigPath), + uriPath: viper.GetString(BkDataUriPathConfigPath), + } + }) + return defaultBkDataApi +} + +func (i *BkDataApi) HttpHeaders(headers map[string]string) http.Header { + headers = i.Headers(headers) + netHeaders := make(http.Header, len(headers)) + for k, v := range headers { + netHeaders[k] = []string{v} + } + return netHeaders +} + +func (i *BkDataApi) Headers(headers map[string]string) map[string]string { + if len(headers) == 0 { + headers = make(map[string]string) + } + headers[BkDataAuthorization] = fmt.Sprintf( + `{"bkdata_authentication_method": "%s", "bkdata_data_token": "%s", "bk_username": "%s"}`, + i.authenticationMethod, i.token, AdminUserName, + ) + return i.bkApi.Headers(headers) +} + +func (i *BkDataApi) url(path string) string { + url := i.bkApi.Url(i.uriPath) + if path != "" { + url = fmt.Sprintf("%s/%s", url, path) + } + return url +} + +func (i *BkDataApi) QueryAsyncUrl() string { + return i.url(QueryAsync) +} + +func (i *BkDataApi) QuerySyncUrl() string { + return i.url(QuerySync) +} + +func (i *BkDataApi) QueryEsUrl() string { + return fmt.Sprintf("%s/es", i.QuerySyncUrl()) +} diff --git a/pkg/unify-query/bkapi/settings.go b/pkg/unify-query/bkapi/settings.go new file mode 100644 index 000000000..51491c016 --- /dev/null +++ b/pkg/unify-query/bkapi/settings.go @@ -0,0 +1,22 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package bkapi + +const ( + // BKDATA 配置 + BkDataUriPathConfigPath = "bk_data.uri_path" + BkDataAuthenticationMethodConfigPath = "bk_data.authentication_method" + BkDataTokenConfigPath = "bk_data.token" + + // BKAPI 配置 + BkApiAddressConfigPath = "bk_api.address" + BkApiCodeConfigPath = "bk_api.code" + BkApiSecretConfigPath = "bk_api.secret" +) diff --git a/pkg/unify-query/cmdb/v1beta1/v1beta1.go b/pkg/unify-query/cmdb/v1beta1/v1beta1.go index 2fad93a8c..bbf1f6ac5 100644 --- a/pkg/unify-query/cmdb/v1beta1/v1beta1.go +++ b/pkg/unify-query/cmdb/v1beta1/v1beta1.go @@ -271,8 +271,8 @@ func (r *model) doRequest(ctx context.Context, lookBackDeltaStr, spaceUid string } metadata.SetExpand(ctx, vmExpand) - instance = prometheus.GetInstance(ctx, &metadata.Query{ - StorageID: consul.VictoriaMetricsStorageType, + instance = prometheus.GetTsDbInstance(ctx, &metadata.Query{ + StorageType: consul.VictoriaMetricsStorageType, }) if instance == nil { err = fmt.Errorf("%s storage get error", consul.VictoriaMetricsStorageType) diff --git a/pkg/unify-query/cmdb/v1beta1/v1beta1_test.go b/pkg/unify-query/cmdb/v1beta1/v1beta1_test.go index 14176a6f1..78fd648cd 100644 --- a/pkg/unify-query/cmdb/v1beta1/v1beta1_test.go +++ b/pkg/unify-query/cmdb/v1beta1/v1beta1_test.go @@ -182,14 +182,12 @@ func mockData(ctx context.Context) *curl.TestCurl { tsdb.SetStorage(consul.VictoriaMetricsStorageType, &tsdb.Storage{ Type: consul.VictoriaMetricsStorageType, Instance: &victoriaMetrics.Instance{ - Ctx: ctx, - Address: "victoria_metric", - UriPath: "api", - Curl: mockCurl, - InfluxCompatible: true, - AuthenticationMethod: "token", + ctx: ctx, + Curl: mockCurl, + InfluxCompatible: true, }, }) + tsdb.SetStorage(influxdbStorageID, &tsdb.Storage{ Type: consul.InfluxDBStorageType, Instance: tsdbInfluxdb.NewInstance( diff --git a/pkg/unify-query/docs/docs.go b/pkg/unify-query/docs/docs.go index 3af98861a..3020a07af 100644 --- a/pkg/unify-query/docs/docs.go +++ b/pkg/unify-query/docs/docs.go @@ -1255,7 +1255,7 @@ const docTemplate = `{ "example": false }, "limit": { - "description": "Limit 点数限制数量", + "description": "MaxLimit 点数限制数量", "type": "integer", "example": 0 }, diff --git a/pkg/unify-query/metadata/reference_test.go b/pkg/unify-query/metadata/reference_test.go index d4ebb56b8..814bfdbf8 100644 --- a/pkg/unify-query/metadata/reference_test.go +++ b/pkg/unify-query/metadata/reference_test.go @@ -35,7 +35,7 @@ func TestQueryReference(t *testing.T) { expected string }{ "default-1": { - refString: `{"a":{"QueryList":[{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"datalink_stats.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"datalink_stats","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"__name__=\"process_start_time_seconds_value\"","VmConditionNum":1,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_dbm_redis_exporter_murphyy_t.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_dbm_redis_exporter_murphyy_t","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_diskkimmy.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_diskkimmy","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"vm-default","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1572865.__default__","VmRt":"2_bkbase_bcs_custom_metrics","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1572865","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"result_table_id=\"2_bkbase_bcs_custom_metrics\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_dbm_redis_exporter_murphy_test.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_dbm_redis_exporter_murphy_test","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"vm-default","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1572864.__default__","VmRt":"2_bcs_prom_computation_result_table","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1572864","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"result_table_id=\"2_bcs_prom_computation_result_table\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1573177.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1573177","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"__name__=\"process_start_time_seconds_value\"","VmConditionNum":1,"Filters":null,"OffsetInfo":{"OffSet":0,"Limit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}}],"ReferenceName":"a","MetricName":"process_start_time_seconds","IsCount":false}}`, + refString: `{"a":{"QueryList":[{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"datalink_stats.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"datalink_stats","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"__name__=\"process_start_time_seconds_value\"","VmConditionNum":1,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_dbm_redis_exporter_murphyy_t.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_dbm_redis_exporter_murphyy_t","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_diskkimmy.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_diskkimmy","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"vm-default","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1572865.__default__","VmRt":"2_bkbase_bcs_custom_metrics","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1572865","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"result_table_id=\"2_bkbase_bcs_custom_metrics\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"exporter_dbm_redis_exporter_murphy_test.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"exporter_dbm_redis_exporter_murphy_test","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"bk_biz_id='2'","BkSqlCondition":"","VmCondition":"bk_biz_id=\"2\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[[{"DimensionName":"bk_biz_id","Value":["2"],"Operator":"contains"}]],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"vm-default","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1572864.__default__","VmRt":"2_bcs_prom_computation_result_table","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1572864","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"result_table_id=\"2_bcs_prom_computation_result_table\", __name__=\"process_start_time_seconds_value\"","VmConditionNum":2,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}},{"SourceType":"","Password":"","ClusterID":"","StorageType":"","StorageID":"2","StorageName":"","ClusterName":"default","TagsKey":null,"TableID":"2_bkmonitor_time_series_1573177.__default__","VmRt":"","IsSingleMetric":true,"RetentionPolicy":"","DB":"2_bkmonitor_time_series_1573177","Measurement":"process_start_time_seconds","Field":"value","Timezone":"UTC","Fields":["value"],"Measurements":["process_start_time_seconds"],"IsHasOr":false,"Aggregates":[],"Condition":"","BkSqlCondition":"","VmCondition":"__name__=\"process_start_time_seconds_value\"","VmConditionNum":1,"Filters":null,"OffsetInfo":{"OffSet":0,"MaxLimit":0,"SOffSet":0,"SLimit":0},"SegmentedEnable":false,"DataSource":"bkmonitor","AllConditions":[],"Source":null,"From":0,"Size":0,"Orders":{}}],"ReferenceName":"a","MetricName":"process_start_time_seconds","IsCount":false}}`, }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 206fe2b2a..318e03fe3 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -351,7 +351,6 @@ func (qRef QueryReference) CheckVmQuery(ctx context.Context) (bool, *VmExpand, e span.Set(fmt.Sprintf("result_table_%s_num", referenceName), len(reference.QueryList)) vmConditions := make(map[string]struct{}) - for _, query := range reference.QueryList { span.Set(fmt.Sprintf("query-%s-is-single-metric", query.TableID), query.IsSingleMetric) diff --git a/pkg/unify-query/query/interfaces.go b/pkg/unify-query/query/interfaces.go index 786e98a64..368c95f6f 100644 --- a/pkg/unify-query/query/interfaces.go +++ b/pkg/unify-query/query/interfaces.go @@ -41,6 +41,10 @@ type TsDBV2 struct { TimeField metadata.TimeField `json:"time_field"` // NeedAddTime NeedAddTime bool `json:"need_add_time"` + + // SourceType 数据来源 + SourceType string `json:"source_type"` + StorageType string `json:"storage_type"` } func (z *TsDBV2) IsSplit() bool { diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index c1bd665c9..81ab42c26 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -440,7 +440,7 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q } qry := &metadata.Query{ - StorageID: consul.BkSqlStorageType, + StorageType: consul.BkSqlStorageType, DB: route.DB(), Measurement: route.Measurement(), Field: q.FieldName, @@ -573,7 +573,6 @@ func (q *Query) BuildMetadataQuery( } if len(queryConditions) > 0 { - // influxdb 查询特殊处理逻辑 influxdbConditions := ConvertToPromBuffer(queryConditions) if len(influxdbConditions) > 0 { @@ -717,6 +716,11 @@ func (q *Query) BuildMetadataQuery( query.StorageID = storageID } + // 判断 rt 是否是 bkdata 的数据源 + if tsDB.SourceType == BkData { + query.StorageType = consul.ElasticsearchStorageType + } + query.TableID = tsDB.TableID query.ClusterName = clusterName query.TagsKey = tagKeys @@ -730,6 +734,7 @@ func (q *Query) BuildMetadataQuery( query.Measurements = measurements query.TimeField = tsDB.TimeField query.NeedAddTime = tsDB.NeedAddTime + query.SourceType = tsDB.SourceType query.Condition = whereList.String() query.VmCondition, query.VmConditionNum = allCondition.VMString(vmRt, vmMetric, q.IsRegexp) diff --git a/pkg/unify-query/query/structured/space.go b/pkg/unify-query/query/structured/space.go index c5caad70e..cd0386f53 100644 --- a/pkg/unify-query/query/structured/space.go +++ b/pkg/unify-query/query/structured/space.go @@ -148,6 +148,7 @@ func (s *SpaceFilter) NewTsDBs(spaceTable *routerInfluxdb.SpaceResultTable, fiel Unit: rtDetail.Options.TimeField.Unit, }, NeedAddTime: rtDetail.Options.NeedAddTime, + SourceType: rtDetail.SourceType, } // 字段为空时，需要返回结果表的信息，表示无需过滤字段过滤 // bklog 或者 bkapm 则不判断 field 是否存在 diff --git a/pkg/unify-query/service/http/check_handler.go b/pkg/unify-query/service/http/check_handler.go index 08c4a8775..49edcdde8 100644 --- a/pkg/unify-query/service/http/check_handler.go +++ b/pkg/unify-query/service/http/check_handler.go @@ -187,7 +187,7 @@ func checkQueryTs(ctx context.Context, q *structured.QueryTs, r *CheckResponse) } else { for _, qm := range qr { for _, qry := range qm.QueryList { - instance := prometheus.GetInstance(ctx, qry) + instance := prometheus.GetTsDbInstance(ctx, qry) if instance == nil { r.Error("prometheus.GetInstance", fmt.Errorf("instance is null, with storageID %s", qry.StorageID)) continue diff --git a/pkg/unify-query/service/http/promql.go b/pkg/unify-query/service/http/promql.go index f78d0c6e9..13272ea5f 100644 --- a/pkg/unify-query/service/http/promql.go +++ b/pkg/unify-query/service/http/promql.go @@ -41,6 +41,7 @@ func NewPromData(dimensions []string) *PromData { } return &PromData{ dimensions: dimensionsMap, + Tables: make([]*TablesItem, 0), } } diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 6e9d39de9..0fafa9a4b 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -87,7 +87,7 @@ func queryExemplar(ctx context.Context, query *structured.QueryTs) (interface{}, for _, qry := range queryMetric.QueryList { qry.Timezone = timezone - instance := prometheus.GetInstance(ctx, qry) + instance := prometheus.GetTsDbInstance(ctx, qry) if instance != nil { res, err := instance.QueryExemplar(ctx, qList.FieldList, qry, start, end) if err != nil { @@ -432,6 +432,10 @@ func queryTsWithPromEngine(ctx context.Context, query *structured.QueryTs) (inte log.Errorf(ctx, fmt.Sprintf("check vm query: %s", err.Error())) } if ok { + if len(vmExpand.ResultTableList) == 0 { + return resp, nil + } + // 函数替换逻辑有问题、暂时屏蔽 // vm 跟 prom 的函数有差异，需要转换一下以完全适配 prometheus。 // https://docs.victoriametrics.com/metricsql/#delta @@ -444,8 +448,8 @@ func queryTsWithPromEngine(ctx context.Context, query *structured.QueryTs) (inte // return nil, err //} metadata.SetExpand(ctx, vmExpand) - instance = prometheus.GetInstance(ctx, &metadata.Query{ - StorageID: consul.VictoriaMetricsStorageType, + instance = prometheus.GetTsDbInstance(ctx, &metadata.Query{ + StorageType: consul.VictoriaMetricsStorageType, }) if instance == nil { err = fmt.Errorf("%s storage get error", consul.VictoriaMetricsStorageType) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index aef224d24..c85404d97 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -381,7 +381,7 @@ func mockData(ctx context.Context, path, bucket string) *curl.TestCurl { tsdb.SetStorage(consul.VictoriaMetricsStorageType, &tsdb.Storage{ Type: consul.VictoriaMetricsStorageType, Instance: &victoriaMetrics.Instance{ - Ctx: ctx, + ctx: ctx, Address: "victoria_metric", UriPath: "api", Curl: mockCurl, @@ -393,7 +393,7 @@ func mockData(ctx context.Context, path, bucket string) *curl.TestCurl { tsdb.SetStorage(strconv.FormatInt(victoriaMetricsStorageId, 10), &tsdb.Storage{ Type: consul.VictoriaMetricsStorageType, Instance: &victoriaMetrics.Instance{ - Ctx: ctx, + ctx: ctx, Address: "victoria_metric", UriPath: "api", Curl: mockCurl, diff --git a/pkg/unify-query/service/http/tsquery_test.go b/pkg/unify-query/service/http/tsquery_test.go index 7cce2b8eb..c3f8c7b83 100644 --- a/pkg/unify-query/service/http/tsquery_test.go +++ b/pkg/unify-query/service/http/tsquery_test.go @@ -132,7 +132,7 @@ func MockTsDB(t *testing.T) { tsdb.SetStorage("10", &tsdb.Storage{ Type: consul.VictoriaMetricsStorageType, Instance: &victoriaMetrics.Instance{ - Ctx: context.TODO(), + ctx: context.TODO(), Address: "127.0.0.1", UriPath: "api", Timeout: time.Minute, diff --git a/pkg/unify-query/service/tsdb/service.go b/pkg/unify-query/service/tsdb/service.go index a9da48852..a1b8a988c 100644 --- a/pkg/unify-query/service/tsdb/service.go +++ b/pkg/unify-query/service/tsdb/service.go @@ -14,13 +14,9 @@ import ( "sync" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" inner "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/offlineDataArchive" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/victoriaMetrics" ) type Service struct { @@ -133,10 +129,6 @@ func (s *Service) reloadStorage() error { } options := &inner.Options{ - VM: &inner.VMOption{ - UriPath: "select/0/prometheus/api/v1", - Timeout: VmTimeout, - }, InfluxDB: &inner.InfluxDBOption{ Timeout: InfluxDBTimeout, ContentType: InfluxDBContentType, @@ -162,67 +154,5 @@ func (s *Service) reloadStorage() error { return err } - inner.SetStorage(consul.OfflineDataArchive, &inner.Storage{ - Type: consul.OfflineDataArchive, - Instance: &offlineDataArchive.Instance{ - Ctx: s.ctx, - Address: OfflineDataArchiveAddress, - Timeout: OfflineDataArchiveTimeout, - MaxLimit: InfluxDBMaxLimit, - MaxSLimit: InfluxDBMaxSLimit, - Toleration: InfluxDBTolerance, - ReadRateLimit: InfluxDBQueryReadRateLimit, - GrpcMaxCallRecvMsgSize: OfflineDataArchiveGrpcMaxCallRecvMsgSize, - GrpcMaxCallSendMsgSize: OfflineDataArchiveGrpcMaxCallSendMsgSize, - }, - }) - - // 增加全局 vm storage 查询 - inner.SetStorage(consul.VictoriaMetricsStorageType, &inner.Storage{ - Type: consul.VictoriaMetricsStorageType, - Instance: &victoriaMetrics.Instance{ - Ctx: s.ctx, - ContentType: VmContentType, - Address: VmAddress, - UriPath: VmUriPath, - Code: VmCode, - Secret: VmSecret, - Token: VmToken, - AuthenticationMethod: VmAuthenticationMethod, - Timeout: VmTimeout, - // 是否开启 influxdb 正则匹配 - MaxConditionNum: VmMaxConditionNum, - InfluxCompatible: VmInfluxCompatible, - UseNativeOr: VmUseNativeOr, - Curl: &curl.HttpCurl{Log: log.DefaultLogger}, - }, - }) - - // 增加 bksql tspider storage 查询 - inner.SetStorage(consul.BkSqlStorageType, &inner.Storage{ - Type: consul.BkSqlStorageType, - Instance: &bksql.Instance{ - Ctx: s.ctx, - Timeout: BkSqlTimeout, - IntervalTime: BkSqlIntervalTime, - Limit: BkSqlLimit, - Tolerance: BkSqlTolerance, - Client: &bksql.Client{ - Address: BkSqlAddress, - BkdataAuthenticationMethod: BkSqlAuthenticationMethod, - BkUsername: bksql.BkUserName, - BkAppCode: BkSqlCode, - BkdataDataToken: BkSqlToken, - BkAppSecret: BkSqlSecret, - ContentType: BkSqlContentType, - Log: log.DefaultLogger, - Timeout: BkSqlTimeout, - Curl: &curl.HttpCurl{ - Log: log.DefaultLogger, - }, - }, - }, - }) - return nil } diff --git a/pkg/unify-query/tsdb/bksql/client.go b/pkg/unify-query/tsdb/bksql/client.go index aedf0c6b6..93cf8368b 100644 --- a/pkg/unify-query/tsdb/bksql/client.go +++ b/pkg/unify-query/tsdb/bksql/client.go @@ -12,48 +12,42 @@ package bksql import ( "context" "encoding/json" - "fmt" "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" ) type Client struct { - Address string + url string + headers map[string]string - BkdataAuthenticationMethod string - BkUsername string - BkAppCode string - - // 不传这个值，bk-sql 会自动筛选 - PreferStorage string - - BkdataDataToken string - BkAppSecret string + curl curl.Curl +} - ContentType string +func (c *Client) WithCurl(cc curl.Curl) *Client { + c.curl = cc + return c +} - Log *log.Logger +func (c *Client) WithUrl(url string) *Client { + c.url = url + return c +} - Timeout time.Duration - Curl curl.Curl +func (c *Client) WithHeader(headers map[string]string) *Client { + c.headers = headers + return c } -func (c *Client) curl(ctx context.Context, method, url, sql string, res *Result, span *trace.Span) error { +func (c *Client) curlGet(ctx context.Context, method, sql string, res *Result, span *trace.Span) error { if method == "" { method = curl.Post } - params := &Params{ - BkdataAuthenticationMethod: c.BkdataAuthenticationMethod, - BkAppCode: c.BkAppCode, - PreferStorage: c.PreferStorage, - BkdataDataToken: c.BkdataDataToken, - } + params := &Params{} if sql != "" { params.SQL = sql @@ -64,19 +58,13 @@ func (c *Client) curl(ctx context.Context, method, url, sql string, res *Result, return err } - ctx, cancel := context.WithTimeout(ctx, c.Timeout) - defer cancel() - startAnaylize := time.Now() - size, err := c.Curl.Request( + size, err := c.curl.Request( ctx, method, curl.Options{ - UrlPath: url, + UrlPath: c.url, Body: body, - Headers: map[string]string{ - ContentType: c.ContentType, - Authorization: c.authorization(), - }, + Headers: c.headers, }, res, ) @@ -88,7 +76,9 @@ func (c *Client) curl(ctx context.Context, method, url, sql string, res *Result, metric.TsDBRequestBytes(ctx, size, user.SpaceUid, user.Source, consul.BkSqlStorageType) queryCost := time.Since(startAnaylize) - span.Set("query-cost", queryCost.String()) + if span != nil { + span.Set("query-cost", queryCost.String()) + } metric.TsDBRequestSecond( ctx, queryCost, user.SpaceUid, consul.BkSqlStorageType, @@ -96,22 +86,11 @@ func (c *Client) curl(ctx context.Context, method, url, sql string, res *Result, return nil } -func (c *Client) authorization() string { - auth := fmt.Sprintf( - `{"bk_username": "%s", "bk_app_code": "%s", "bk_app_secret": "%s"}`, - c.BkUsername, - c.BkAppCode, - c.BkAppSecret, - ) - return auth -} - func (c *Client) QuerySync(ctx context.Context, sql string, span *trace.Span) *Result { data := &QuerySyncResultData{} res := c.response(data) - url := fmt.Sprintf("%s/%s", c.Address, QuerySync) - err := c.curl(ctx, curl.Post, url, sql, res, span) + err := c.curlGet(ctx, curl.Post, sql, res, span) if err != nil { return c.failed(ctx, err) } @@ -119,50 +98,11 @@ func (c *Client) QuerySync(ctx context.Context, sql string, span *trace.Span) *R return res } -func (c *Client) QueryAsync(ctx context.Context, sql string, span *trace.Span) *Result { - data := &QueryAsyncData{} - res := c.response(data) - - url := fmt.Sprintf("%s/%s", c.Address, QueryAsync) - err := c.curl(ctx, curl.Post, url, sql, res, span) - if err != nil { - return c.failed(ctx, err) - } - - return res -} - -func (c *Client) QueryAsyncResult(ctx context.Context, queryID string, span *trace.Span) *Result { - data := &QueryAsyncResultData{} - res := c.response(data) - - url := fmt.Sprintf("%s/%s/result/%s", c.Address, QueryAsync, queryID) - err := c.curl(ctx, curl.Get, url, "", res, span) - if err != nil { - return c.failed(ctx, err) - } - - return res -} - -func (c *Client) QueryAsyncState(ctx context.Context, queryID string, span *trace.Span) *Result { - data := &QueryAsyncStateData{} - res := c.response(data) - - url := fmt.Sprintf("%s/%s/state/%s", c.Address, QueryAsync, queryID) - err := c.curl(ctx, curl.Get, url, "", res, span) - if err != nil { - return c.failed(ctx, err) - } - return res -} - func (c *Client) response(data interface{}) *Result { return &Result{Data: data} } func (c *Client) failed(ctx context.Context, err error) *Result { - c.Log.Errorf(ctx, err.Error()) return &Result{ Result: false, Message: err.Error(), diff --git a/pkg/unify-query/tsdb/bksql/client_test.go b/pkg/unify-query/tsdb/bksql/client_test.go index e8f018171..c5e97c4c8 100644 --- a/pkg/unify-query/tsdb/bksql/client_test.go +++ b/pkg/unify-query/tsdb/bksql/client_test.go @@ -21,8 +21,8 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" ) var ( @@ -44,19 +44,8 @@ func mockClient() *Client { token = viper.GetString("mock.bk_sql.token") client = &Client{ - Address: address, - BkdataAuthenticationMethod: "token", - BkUsername: BkUserName, - BkAppCode: code, - PreferStorage: TSpider, - BkdataDataToken: token, - BkAppSecret: secret, - Timeout: time.Second * 30, - Log: log.DefaultLogger, - ContentType: "application/json", - Curl: &curl.HttpCurl{ - Log: log.DefaultLogger, - }, + url: address, + curl: &curl.HttpCurl{Log: log.DefaultLogger}, } }) return client @@ -68,7 +57,7 @@ func TestClient_QuerySync(t *testing.T) { mock.Init() mockClient() - ctx, span := trace.NewSpan(ctx, "test_client") + ctx = metadata.InitHashID(ctx) end := time.Now() start := end.Add(time.Minute * -5) @@ -80,7 +69,7 @@ func TestClient_QuerySync(t *testing.T) { start.UnixMilli(), end.UnixMilli(), ), - span, + nil, ) assert.Equal(t, StatusOK, res.Code) @@ -92,89 +81,3 @@ func TestClient_QuerySync(t *testing.T) { log.Infof(ctx, "%+v", d) } } - -func TestClient_QueryAsync(t *testing.T) { - ctx := context.Background() - - mock.Init() - mockClient() - - ctx, span := trace.NewSpan(ctx, "test_client") - - end := time.Now() - start := end.Add(time.Minute * -5) - - res := client.QueryAsync( - ctx, - fmt.Sprintf( - `SELECT * FROM 132_hander_opmon_avg WHERE dtEventTimeStamp >= %d AND dtEventTimeStamp < %d LIMIT 10`, - start.UnixMilli(), - end.UnixMilli(), - ), - span, - ) - - assert.Equal(t, StatusOK, res.Code) - d, ok := res.Data.(*QueryAsyncData) - assert.True(t, ok) - - if d != nil { - assert.NotEmpty(t, d.QueryId) - log.Infof(ctx, "%+v", d) - } -} - -func TestClient_QueryAsyncState(t *testing.T) { - ctx := context.Background() - - mock.Init() - mockClient() - - ctx, span := trace.NewSpan(ctx, "test_client") - - end := time.Now() - start := end.Add(time.Minute * -5) - - res := client.QueryAsync( - ctx, - fmt.Sprintf( - `SELECT * FROM 132_hander_opmon_avg WHERE dtEventTimeStamp >= %d AND dtEventTimeStamp < %d LIMIT 10`, - start.UnixMilli(), - end.UnixMilli(), - ), - span, - ) - - assert.Equal(t, StatusOK, res.Code) - data := res.Data.(*QueryAsyncData) - if data != nil { - r := client.QueryAsyncState(ctx, data.QueryId, span) - - assert.Equal(t, r.Code, StatusOK) - d, ok := r.Data.(*QueryAsyncStateData) - assert.True(t, ok) - - if d != nil { - assert.NotEmpty(t, d.State) - log.Infof(ctx, "%+v", d) - } - } -} - -func TestClient_QueryAsyncResult(t *testing.T) { - ctx := context.Background() - - mock.Init() - mockClient() - - res := client.QueryAsyncResult(ctx, "BK912760164455546880", nil) - - assert.Equal(t, res.Code, StatusOK) - d, ok := res.Data.(*QueryAsyncResultData) - assert.True(t, ok) - - if d != nil { - assert.NotEmpty(t, d.List) - log.Infof(ctx, "%+v", d) - } -} diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 8e7d52880..6ade4c419 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -22,24 +22,29 @@ import ( "github.com/prometheus/prometheus/storage/remote" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" ) type Instance struct { - Ctx context.Context + ctx context.Context - Timeout time.Duration - IntervalTime time.Duration + querySyncUrl string + queryAsyncUrl string + + headers map[string]string + + timeout time.Duration + intervalTime time.Duration - Limit int - Tolerance int + maxLimit int + tolerance int - Client *Client + client *Client } func (i *Instance) QueryReference(ctx context.Context, query *metadata.Query, start int64, end int64) (*prompb.QueryResult, error) { @@ -49,6 +54,33 @@ func (i *Instance) QueryReference(ctx context.Context, query *metadata.Query, st var _ tsdb.Instance = (*Instance)(nil) +type Options struct { + Address string + Headers map[string]string + + Timeout time.Duration + IntervalTime time.Duration + MaxLimit int + Tolerance int + + Curl curl.Curl +} + +func NewInstance(ctx context.Context, opt Options) (*Instance, error) { + if opt.Address == "" { + return nil, fmt.Errorf("address is empty") + } + instance := &Instance{ + ctx: ctx, + timeout: opt.Timeout, + intervalTime: opt.IntervalTime, + maxLimit: opt.MaxLimit, + tolerance: opt.Tolerance, + client: (&Client{}).WithUrl(opt.Address).WithHeader(opt.Headers).WithCurl(opt.Curl), + } + return instance, nil +} + func (i *Instance) checkResult(res *Result) error { if !res.Result { return fmt.Errorf( @@ -80,17 +112,17 @@ func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) ( log.Infof(ctx, "%s: %s", i.GetInstanceType(), sql) span.Set("query-sql", sql) - ctx, cancel := context.WithTimeout(ctx, i.Timeout) + ctx, cancel := context.WithTimeout(ctx, i.timeout) defer cancel() // 发起异步查询 - res := i.Client.QuerySync(ctx, sql, span) + res := i.client.QuerySync(ctx, sql, span) if err = i.checkResult(res); err != nil { return data, err } - span.Set("query-timeout", i.Timeout.String()) - span.Set("query-interval-time", i.IntervalTime.String()) + span.Set("query-timeout", i.timeout.String()) + span.Set("query-interval-time", i.intervalTime.String()) if data, ok = res.Data.(*QuerySyncResultData); !ok { return data, fmt.Errorf("queryAsyncResult type is error: %T", res.Data) @@ -99,92 +131,6 @@ func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) ( return data, nil } -func (i *Instance) queryAsync(ctx context.Context, sql string, span *trace.Span) (*QueryAsyncResultData, error) { - var ( - data *QueryAsyncData - stateData *QueryAsyncStateData - resultData *QueryAsyncResultData - - startAnaylize time.Time - - ok bool - err error - ) - - log.Infof(ctx, "%s: %s", i.GetInstanceType(), sql) - span.Set("query-sql", sql) - - ctx, cancel := context.WithTimeout(ctx, i.Timeout) - defer cancel() - - user := metadata.GetUser(ctx) - startAnaylize = time.Now() - - // 发起异步查询 - res := i.Client.QueryAsync(ctx, sql, span) - if err = i.checkResult(res); err != nil { - return resultData, err - } - - queryCost := time.Since(startAnaylize) - metric.TsDBRequestSecond( - ctx, queryCost, user.SpaceUid, i.GetInstanceType(), - ) - - if data, ok = res.Data.(*QueryAsyncData); !ok { - return resultData, fmt.Errorf("queryAsyncData type is error: %T", res.Data) - } - - if data == nil || data.QueryId == "" { - return resultData, fmt.Errorf("queryAsyncData queryID is emtpy: %+v", data) - } - - span.Set("query-timeout", i.Timeout.String()) - span.Set("query-interval-time", i.IntervalTime.String()) - span.Set("data-query-id", data.QueryId) - - err = func() error { - for { - select { - case <-ctx.Done(): - return fmt.Errorf("queryAsyncState %s timeout %s", data.QueryId, i.Timeout.String()) - default: - stateRes := i.Client.QueryAsyncState(ctx, data.QueryId, span) - if err = i.checkResult(res); err != nil { - return err - } - if stateData, ok = stateRes.Data.(*QueryAsyncStateData); !ok { - return fmt.Errorf("queryAsyncState type is error: %T", res.Data) - } - switch stateData.State { - case RUNNING: - time.Sleep(i.IntervalTime) - continue - case FINISHED: - return nil - default: - return fmt.Errorf("queryAsyncState error %+v", stateData) - } - } - } - }() - - if err != nil { - return resultData, err - } - - resultRes := i.Client.QueryAsyncResult(ctx, data.QueryId, span) - if err = i.checkResult(res); err != nil { - return resultData, err - } - - if resultData, ok = resultRes.Data.(*QueryAsyncResultData); !ok { - return resultData, fmt.Errorf("queryAsyncResult type is error: %T", res.Data) - } - - return resultData, nil -} - func (i *Instance) dims(dims []string, field string) []string { dimensions := make([]string, 0) for _, dim := range dims { @@ -329,7 +275,7 @@ func (i *Instance) bkSql(ctx context.Context, query *metadata.Query, start, end ctx, span := trace.NewSpan(ctx, "bksql-make-sqlBuilder") defer span.End(&err) - maxLimit := i.Limit + i.Tolerance + maxLimit := i.maxLimit + i.tolerance limit := query.Size if limit == 0 || limit > maxLimit { limit = maxLimit @@ -405,14 +351,14 @@ func (i *Instance) query( ctx, span := trace.NewSpan(ctx, "bk-sql-query") defer span.End(&err) - if i.Client == nil { + if i.client == nil { return nil, fmt.Errorf("es client is nil") } - if i.Limit > 0 { - maxLimit := i.Limit + i.Tolerance + if i.maxLimit > 0 { + maxLimit := i.maxLimit + i.tolerance // 如果不传 size，则取最大的限制值 - if query.Size == 0 || query.Size > i.Limit { + if query.Size == 0 || query.Size > i.maxLimit { query.Size = maxLimit } } @@ -443,10 +389,10 @@ func (i *Instance) QueryRaw(ctx context.Context, query *metadata.Query, start, e return storage.ErrSeriesSet(fmt.Errorf("range time is error, start: %s, end: %s ", start, end)) } - if i.Limit > 0 { - maxLimit := i.Limit + i.Tolerance + if i.maxLimit > 0 { + maxLimit := i.maxLimit + i.tolerance // 如果不传 size，则取最大的限制值 - if query.Size == 0 || query.Size > i.Limit { + if query.Size == 0 || query.Size > i.maxLimit { query.Size = maxLimit } } @@ -468,8 +414,8 @@ func (i *Instance) QueryRaw(ctx context.Context, query *metadata.Query, start, e span.Set("data-total-records", data.TotalRecords) log.Infof(ctx, "total records: %d", data.TotalRecords) - if i.Limit > 0 && data.TotalRecords > i.Limit { - return storage.ErrSeriesSet(fmt.Errorf("记录数(%d)超过限制(%d)", data.TotalRecords, i.Limit)) + if i.maxLimit > 0 && data.TotalRecords > i.maxLimit { + return storage.ErrSeriesSet(fmt.Errorf("记录数(%d)超过限制(%d)", data.TotalRecords, i.maxLimit)) } qr, err := i.formatData(start, query.Field, data.SelectFieldsOrder, data.List) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 0f37edb84..f7b885f71 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -34,7 +34,7 @@ func TestInstance_QueryRaw(t *testing.T) { IntervalTime: 3e2 * time.Millisecond, Timeout: 3e1 * time.Second, Client: cli, - Limit: 1e4, + MaxLimit: 1e4, Tolerance: 5, } end := time.Now() @@ -179,7 +179,7 @@ func TestInstance_bkSql(t *testing.T) { } ins := Instance{ - Limit: 2e5, + MaxLimit: 2e5, Tolerance: 5, } diff --git a/pkg/unify-query/tsdb/bksql/struct.go b/pkg/unify-query/tsdb/bksql/struct.go index ddc65246e..63c3c3aee 100644 --- a/pkg/unify-query/tsdb/bksql/struct.go +++ b/pkg/unify-query/tsdb/bksql/struct.go @@ -10,11 +10,7 @@ package bksql const ( - QueryAsync = "query_async" - QuerySync = "query_sync" - - BkUserName = "admin" - TSpider = "tspider" + TSpider = "tspider" StatusOK = "00" StatusFailed = "-1" diff --git a/pkg/unify-query/tsdb/elasticsearch/agg_format.go b/pkg/unify-query/tsdb/elasticsearch/agg_format.go index 96ed0b507..93ddef960 100644 --- a/pkg/unify-query/tsdb/elasticsearch/agg_format.go +++ b/pkg/unify-query/tsdb/elasticsearch/agg_format.go @@ -42,6 +42,8 @@ type aggFormat struct { toEs func(k string) string toProm func(k string) string + timeFormat func(i int64) int64 + dims []string item item items items @@ -132,7 +134,13 @@ func (a *aggFormat) ts(idx int, data elastic.Aggregations) error { } // 时间和值也是不同层级，需要暂存在 a.sample 里 - a.item.timestamp = int64(bucket.Key) + timestamp := int64(bucket.Key) + if a.timeFormat != nil { + a.item.timestamp = a.timeFormat(timestamp) + } else { + a.item.timestamp = timestamp + } + if err := a.ts(idx, bucket.Aggregations); err != nil { return err } diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 5a5b6bf22..38bbd8d5f 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -150,6 +150,8 @@ type FormatFactory struct { toEs func(k string) string toProm func(k string) string + timeFormat func(i int64) int64 + mapping map[string]string data map[string]any @@ -204,6 +206,13 @@ func (f *FormatFactory) WithQuery(valueKey string, timeField metadata.TimeField, timeField.UnitRate = 1 } + // 如果是 long 类型的话，需要转换为毫秒格式 + if timeField.Type == TimeFieldTypeInt { + f.timeFormat = func(i int64) int64 { + return i / timeField.UnitRate * 1e3 + } + } + // 根据时间单位把考试时间和结束实践转换为对应值 f.start = start f.end = end @@ -212,6 +221,7 @@ func (f *FormatFactory) WithQuery(valueKey string, timeField metadata.TimeField, f.timeField = timeField f.from = from f.size = size + return f } @@ -329,6 +339,7 @@ func (f *FormatFactory) AggDataFormat(data elastic.Aggregations) (map[string]*pr items: make(items, 0), toEs: f.toEs, toProm: f.toProm, + timeFormat: f.timeFormat, } af.get() @@ -702,13 +713,20 @@ func (f *FormatFactory) Sample() (prompb.Sample, error) { } if timestamp, ok = f.data[f.timeField.Name]; ok { + fact := 1e3 / f.timeField.UnitRate switch timestamp.(type) { case int64: - sample.Timestamp = timestamp.(int64) * 1e3 + sample.Timestamp = timestamp.(int64) * fact case int: - sample.Timestamp = int64(timestamp.(int) * 1e3) + sample.Timestamp = int64(timestamp.(int)) * fact + case float64: + sample.Timestamp = int64(timestamp.(float64)) * fact case string: - sample.Timestamp, err = strconv.ParseInt(timestamp.(string), 10, 64) + v, parseErr := strconv.ParseInt(timestamp.(string), 10, 64) + if parseErr != nil { + return sample, parseErr + } + sample.Timestamp = v * fact default: return sample, fmt.Errorf("timestamp key type is error: %T, %v", timestamp, timestamp) } diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 409c8716a..7cf32a8a4 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -14,6 +14,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "sort" "strings" "sync" @@ -86,12 +87,14 @@ func (i *Instance) GetInstanceType() string { } type InstanceOption struct { - Address string - Username string - Password string - MaxSize int - MaxRouting int - Timeout time.Duration + Address string + Username string + Password string + MaxSize int + MaxRouting int + Timeout time.Duration + Headers http.Header + HealthCheck bool } type queryOption struct { @@ -117,7 +120,6 @@ var TimeSeriesResultPool = sync.Pool{ } func NewInstance(ctx context.Context, opt *InstanceOption) (*Instance, error) { - ins := &Instance{ ctx: ctx, timeout: opt.Timeout, @@ -133,9 +135,17 @@ func NewInstance(ctx context.Context, opt *InstanceOption) (*Instance, error) { cliOpts := []elastic.ClientOptionFunc{ elastic.SetURL(opt.Address), elastic.SetSniff(false), + elastic.SetHealthcheck(opt.HealthCheck), + } + if len(opt.Headers) > 0 { + cliOpts = append(cliOpts, elastic.SetHeaders(opt.Headers)) } + if opt.Username != "" && opt.Password != "" { - cliOpts = append(cliOpts, elastic.SetBasicAuth(opt.Username, opt.Password)) + cliOpts = append( + cliOpts, + elastic.SetBasicAuth(opt.Username, opt.Password), + ) } cli, err := elastic.NewClient(cliOpts...) @@ -170,7 +180,7 @@ func (i *Instance) getMappings(ctx context.Context, aliases []string) ([]map[str }() span.Set("alias", aliases) - mappingMap, err := i.client.GetMapping().Index(aliases...).Do(ctx) + mappingMap, err := i.client.GetMapping().Index(aliases...).Type("").Do(ctx) indexes := make([]string, 0, len(mappingMap)) for index := range mappingMap { @@ -585,6 +595,7 @@ func (i *Instance) QueryRaw( mappings, err1 := i.getMappings(ctx, qo.indexes) // index 不存在，mappings 获取异常直接返回空 if len(mappings) == 0 { + log.Warnf(ctx, "index is empty with %v", qo.indexes) return } diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index b0f548cbe..ee3e4c05e 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/bkapi" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" @@ -36,12 +37,17 @@ func TestInstance_queryReference(t *testing.T) { timeout := viper.GetDuration("mock.es.timeout") maxSize := viper.GetInt("mock.es.max_size") maxRouting := viper.GetInt("mock.es.max_routing") + sourceType := viper.GetString("mock.es.source_type") ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() metadata.GetQueryParams(ctx).SetDataSource(structured.BkLog) + if sourceType == "bkdata" { + address = bkapi.GetBkDataApi().QueryEsUrl() + } + ins, err := NewInstance(ctx, &InstanceOption{ Address: address, Username: username, @@ -54,11 +60,18 @@ func TestInstance_queryReference(t *testing.T) { return } - defaultStart := time.UnixMilli(1717027200000) - defaultEnd := time.UnixMilli(1717027230000) + defaultEnd := time.Now() + defaultStart := defaultEnd.Add(time.Hour * -1) + + //db := "2_bklog_bk_unify_query_*_read" + //field := "gseIndex" - db := "2_bklog_bkapigateway_esb_container1_*_read" - field := "gseIndex" + // bkdata db + defaultEnd = time.UnixMilli(1722527999000) + defaultStart = time.UnixMilli(1717171200000) + + db := "39_bklog_bkaudit_plugin_20240723_66b8acde57_202407*" + field := "dtEventTimeStamp" for idx, c := range map[string]struct { query *metadata.Query @@ -89,8 +102,8 @@ func TestInstance_queryReference(t *testing.T) { }, QueryString: "fields.field_name: bk-dev-3", }, - start: time.UnixMilli(1717482000000), - end: time.UnixMilli(1717482160000), + start: defaultStart, + end: defaultEnd, }, "nested aggregate + query 测试": { query: &metadata.Query{ @@ -202,6 +215,11 @@ func TestInstance_queryReference(t *testing.T) { Field: field, From: 0, Size: 10, + TimeField: metadata.TimeField{ + Name: "dtEventTimeStamp", + Type: TimeFieldTypeTime, + Unit: Millisecond, + }, Orders: metadata.Orders{ FieldTime: false, }, diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index e032c6b67..cd1bd19a6 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -85,7 +85,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { leftQ = elastic.NewNestedQuery(key, leftQ) } } else { - leftQ = elastic.NewQueryStringQuery(c.Value) + leftQ = elastic.NewQueryStringQuery(fmt.Sprintf(`"%s"`, c.Value)) } case *parser.NumberRangeCondition: q := elastic.NewRangeQuery(c.Field) @@ -110,7 +110,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { leftQ = elastic.NewNestedQuery(key, leftQ) } } else { - leftQ = elastic.NewQueryStringQuery(c.Value) + leftQ = elastic.NewQueryStringQuery(c.Value).AnalyzeWildcard(true) } default: err = fmt.Errorf("condition type is not match %T", condition) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 433a3be92..40d8baa12 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -80,11 +80,21 @@ func TestQsToDsl(t *testing.T) { }, { q: `"1642903" AND OR NOT "get_proc_status_v2"`, - err: fmt.Errorf("syntax error: unexpected tOR"), + err: fmt.Errorf("query string parser error syntax error: unexpected tOR"), }, { q: `sync_spaces AND -keyword AND -BKLOGAPI`, }, + { + q: "message queue conflict", + }, + { + q: "\"message queue conflict\"", + }, + { + q: `"qu?ck br*wn"`, + expected: `{"bool":{"must":[{"query_string":{"query":"qu?ck"}},{"query_string":{"query":"br*wn"}}]}}`, + }, } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { ctx = metadata.InitHashID(ctx) diff --git a/pkg/unify-query/tsdb/influxdb/instance.go b/pkg/unify-query/tsdb/influxdb/instance.go index 1efbbf040..0ccfb1220 100644 --- a/pkg/unify-query/tsdb/influxdb/instance.go +++ b/pkg/unify-query/tsdb/influxdb/instance.go @@ -64,7 +64,7 @@ var ( ) // NewInstance 初始化引擎 -func NewInstance(ctx context.Context, opt Options) *Instance { +func NewInstance(ctx context.Context, opt Options) (*Instance, error) { headers := map[string]string{} if opt.Accept != "" { headers[ContentType] = opt.Accept @@ -73,7 +73,11 @@ func NewInstance(ctx context.Context, opt Options) *Instance { headers[ContentEncoding] = opt.AcceptEncoding } - return &Instance{ + if opt.Host == "" { + return nil, fmt.Errorf("host is empty %+v", opt) + } + + inst := &Instance{ ctx: ctx, host: opt.Host, port: opt.Port, @@ -98,6 +102,7 @@ func NewInstance(ctx context.Context, opt Options) *Instance { timeout: opt.Timeout, curl: opt.Curl, } + return inst, nil } var _ tsdb.Instance = (*Instance)(nil) diff --git a/pkg/unify-query/tsdb/influxdb/instance_test.go b/pkg/unify-query/tsdb/influxdb/instance_test.go index 9da8af23f..9b77c6415 100644 --- a/pkg/unify-query/tsdb/influxdb/instance_test.go +++ b/pkg/unify-query/tsdb/influxdb/instance_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" ) @@ -92,7 +93,10 @@ func TestInstance_MakeSQL(t *testing.T) { for n, c := range testCases { t.Run(n, func(t *testing.T) { ctx = metadata.InitHashID(ctx) - instance := NewInstance(ctx, option) + instance, err := NewInstance(ctx, option) + if err != nil { + log.Fatalf(ctx, err.Error()) + } if c.query.DB == "" { c.query.DB = db } diff --git a/pkg/unify-query/tsdb/prometheus/instance_test.go b/pkg/unify-query/tsdb/prometheus/instance_test.go index 3be7b5c74..0148ca1a5 100644 --- a/pkg/unify-query/tsdb/prometheus/instance_test.go +++ b/pkg/unify-query/tsdb/prometheus/instance_test.go @@ -321,6 +321,472 @@ func fakeData(ctx context.Context) { }, } + data = []*remote.TimeSeries{ + { + Samples: []*remote.Sample{ + //{Value: 89254.443482, TimestampMs: 1723788900000}, + //{Value: 89271.419844, TimestampMs: 1723788915000}, + //{Value: 89281.17737, TimestampMs: 1723788930000}, + //{Value: 89294.234335, TimestampMs: 1723788945000}, + //{Value: 89310.166, TimestampMs: 1723788960000}, + //{Value: 89321.718902, TimestampMs: 1723788975000}, + //{Value: 89330.943656, TimestampMs: 1723788990000}, + //{Value: 89341.437103, TimestampMs: 1723789005000}, + //{Value: 89356.714587, TimestampMs: 1723789020000}, + //{Value: 89368.57525, TimestampMs: 1723789035000}, + //{Value: 89377.891269, TimestampMs: 1723789050000}, + //{Value: 89389.237067, TimestampMs: 1723789065000}, + //{Value: 89403.894449, TimestampMs: 1723789080000}, + //{Value: 89418.157712, TimestampMs: 1723789095000}, + //{Value: 89429.995274, TimestampMs: 1723789110000}, + //{Value: 89449.490131, TimestampMs: 1723789125000}, + //{Value: 89463.821452, TimestampMs: 1723789140000}, + //{Value: 89463.821452, TimestampMs: 1723789155000}, + //{Value: 89477.877679, TimestampMs: 1723789170000}, + //{Value: 89491.614274, TimestampMs: 1723789185000}, + //{Value: 89514.946665, TimestampMs: 1723789200000}, + //{Value: 89525.187888, TimestampMs: 1723789215000}, + //{Value: 89536.199943, TimestampMs: 1723789230000}, + //{Value: 89546.285583, TimestampMs: 1723789245000}, + //{Value: 89561.879647, TimestampMs: 1723789260000}, + //{Value: 89576.112585, TimestampMs: 1723789275000}, + //{Value: 89587.296897, TimestampMs: 1723789290000}, + //{Value: 89602.584535, TimestampMs: 1723789305000}, + //{Value: 89616.72048, TimestampMs: 1723789320000}, + //{Value: 89629.581803, TimestampMs: 1723789335000}, + //{Value: 89629.581803, TimestampMs: 1723789350000}, + //{Value: 89644.597443, TimestampMs: 1723789365000}, + //{Value: 89661.223251, TimestampMs: 1723789380000}, + //{Value: 89681.051627, TimestampMs: 1723789395000}, + //{Value: 89691.466575, TimestampMs: 1723789410000}, + //{Value: 89691.466575, TimestampMs: 1723789425000}, + //{Value: 89708.374236, TimestampMs: 1723789440000}, + //{Value: 89719.580165, TimestampMs: 1723789455000}, + //{Value: 89734.549785, TimestampMs: 1723789470000}, + //{Value: 89751.197662, TimestampMs: 1723789485000}, + //{Value: 89764.596974, TimestampMs: 1723789500000}, + //{Value: 89775.94415, TimestampMs: 1723789515000}, + //{Value: 89791.728723, TimestampMs: 1723789530000}, + //{Value: 89805.817496, TimestampMs: 1723789545000}, + //{Value: 89817.497365, TimestampMs: 1723789560000}, + //{Value: 89832.928259, TimestampMs: 1723789575000}, + //{Value: 89844.88877, TimestampMs: 1723789590000}, + //{Value: 89853.642183, TimestampMs: 1723789605000}, + //{Value: 89869.9064, TimestampMs: 1723789620000}, + //{Value: 89881.06497, TimestampMs: 1723789635000}, + //{Value: 89897.480595, TimestampMs: 1723789650000}, + //{Value: 89897.480595, TimestampMs: 1723789665000}, + //{Value: 89920.95489, TimestampMs: 1723789680000}, + //{Value: 89933.523817, TimestampMs: 1723789695000}, + //{Value: 89943.552657, TimestampMs: 1723789710000}, + //{Value: 89954.967603, TimestampMs: 1723789725000}, + //{Value: 89967.46319, TimestampMs: 1723789740000}, + //{Value: 89978.93852, TimestampMs: 1723789755000}, + //{Value: 89990.527473, TimestampMs: 1723789770000}, + //{Value: 90003.530246, TimestampMs: 1723789785000}, + //{Value: 90016.504908, TimestampMs: 1723789800000}, + //{Value: 90026.261807, TimestampMs: 1723789815000}, + //{Value: 90040.375899, TimestampMs: 1723789830000}, + //{Value: 90051.124329, TimestampMs: 1723789845000}, + //{Value: 90072.787219, TimestampMs: 1723789860000}, + //{Value: 90083.422473, TimestampMs: 1723789875000}, + //{Value: 90096.186985, TimestampMs: 1723789890000}, + //{Value: 90107.191318, TimestampMs: 1723789905000}, + //{Value: 90122.454399, TimestampMs: 1723789920000}, + //{Value: 90131.422952, TimestampMs: 1723789935000}, + //{Value: 90148.22782, TimestampMs: 1723789950000}, + //{Value: 90159.659718, TimestampMs: 1723789965000}, + //{Value: 90171.257627, TimestampMs: 1723789980000}, + //{Value: 90186.055175, TimestampMs: 1723789995000}, + //{Value: 90194.81363, TimestampMs: 1723790010000}, + //{Value: 90204.743511, TimestampMs: 1723790025000}, + //{Value: 90219.01107, TimestampMs: 1723790040000}, + //{Value: 90229.438247, TimestampMs: 1723790055000}, + //{Value: 90244.301206, TimestampMs: 1723790070000}, + //{Value: 90258.517053, TimestampMs: 1723790085000}, + //{Value: 90274.690343, TimestampMs: 1723790100000}, + //{Value: 90290.162218, TimestampMs: 1723790115000}, + //{Value: 90299.116268, TimestampMs: 1723790130000}, + //{Value: 90309.50359, TimestampMs: 1723790145000}, + //{Value: 90323.006066, TimestampMs: 1723790160000}, + //{Value: 90342.314282, TimestampMs: 1723790175000}, + //{Value: 90354.520428, TimestampMs: 1723790190000}, + //{Value: 90365.376755, TimestampMs: 1723790205000}, + //{Value: 90376.388391, TimestampMs: 1723790220000}, + //{Value: 90389.190165, TimestampMs: 1723790235000}, + //{Value: 90402.389841, TimestampMs: 1723790250000}, + //{Value: 90418.680096, TimestampMs: 1723790265000}, + //{Value: 90432.830444, TimestampMs: 1723790280000}, + //{Value: 90432.830444, TimestampMs: 1723790295000}, + //{Value: 90448.193001, TimestampMs: 1723790310000}, + //{Value: 90463.955856, TimestampMs: 1723790325000}, + //{Value: 90473.328555, TimestampMs: 1723790340000}, + //{Value: 90485.445743, TimestampMs: 1723790355000}, + //{Value: 90500.308975, TimestampMs: 1723790370000}, + //{Value: 90513.996363, TimestampMs: 1723790385000}, + //{Value: 90524.583692, TimestampMs: 1723790400000}, + //{Value: 90539.642466, TimestampMs: 1723790415000}, + //{Value: 90554.297539, TimestampMs: 1723790430000}, + //{Value: 90568.418917, TimestampMs: 1723790445000}, + //{Value: 90578.465894, TimestampMs: 1723790460000}, + //{Value: 90590.308212, TimestampMs: 1723790475000}, + //{Value: 90610.786895, TimestampMs: 1723790490000}, + //{Value: 90619.853563, TimestampMs: 1723790505000}, + //{Value: 90633.344401, TimestampMs: 1723790520000}, + //{Value: 90643.642528, TimestampMs: 1723790535000}, + //{Value: 90654.855284, TimestampMs: 1723790550000}, + //{Value: 90671.192098, TimestampMs: 1723790565000}, + //{Value: 90685.384962, TimestampMs: 1723790580000}, + //{Value: 90695.301196, TimestampMs: 1723790595000}, + //{Value: 90706.41933, TimestampMs: 1723790610000}, + //{Value: 90717.570439, TimestampMs: 1723790625000}, + //{Value: 90730.782064, TimestampMs: 1723790640000}, + //{Value: 90740.953431, TimestampMs: 1723790655000}, + //{Value: 90757.409502, TimestampMs: 1723790670000}, + //{Value: 90769.1984, TimestampMs: 1723790685000}, + //{Value: 90783.309249, TimestampMs: 1723790700000}, + //{Value: 90797.414003, TimestampMs: 1723790715000}, + //{Value: 90807.548935, TimestampMs: 1723790730000}, + //{Value: 90820.106166, TimestampMs: 1723790745000}, + //{Value: 90835.907679, TimestampMs: 1723790760000}, + //{Value: 90846.165194, TimestampMs: 1723790775000}, + //{Value: 90863.011773, TimestampMs: 1723790790000}, + //{Value: 90874.307508, TimestampMs: 1723790805000}, + //{Value: 90889.763538, TimestampMs: 1723790820000}, + //{Value: 90900.335085, TimestampMs: 1723790835000}, + //{Value: 90908.988402, TimestampMs: 1723790850000}, + //{Value: 90920.469334, TimestampMs: 1723790865000}, + //{Value: 90933.826482, TimestampMs: 1723790880000}, + //{Value: 90947.39323, TimestampMs: 1723790895000}, + //{Value: 90963.091511, TimestampMs: 1723790910000}, + //{Value: 90977.368179, TimestampMs: 1723790925000}, + //{Value: 90986.163197, TimestampMs: 1723790940000}, + //{Value: 90995.061955, TimestampMs: 1723790955000}, + //{Value: 91006.194755, TimestampMs: 1723790970000}, + //{Value: 91029.735577, TimestampMs: 1723790985000}, + //{Value: 91029.735577, TimestampMs: 1723791000000}, + //{Value: 91044.818113, TimestampMs: 1723791015000}, + //{Value: 91059.875089, TimestampMs: 1723791030000}, + //{Value: 91080.865333, TimestampMs: 1723791045000}, + //{Value: 91090.649849, TimestampMs: 1723791060000}, + //{Value: 91104.369272, TimestampMs: 1723791075000}, + //{Value: 91117.172565, TimestampMs: 1723791090000}, + //{Value: 91131.69536, TimestampMs: 1723791105000}, + //{Value: 91131.69536, TimestampMs: 1723791120000}, + //{Value: 91148.506416, TimestampMs: 1723791135000}, + //{Value: 91170.250174, TimestampMs: 1723791150000}, + //{Value: 91180.622433, TimestampMs: 1723791165000}, + //{Value: 91193.951386, TimestampMs: 1723791180000}, + //{Value: 91203.445343, TimestampMs: 1723791195000}, + //{Value: 91216.939029, TimestampMs: 1723791210000}, + //{Value: 91225.686877, TimestampMs: 1723791225000}, + //{Value: 91241.017744, TimestampMs: 1723791240000}, + //{Value: 91250.610082, TimestampMs: 1723791255000}, + //{Value: 91266.805103, TimestampMs: 1723791270000}, + //{Value: 91280.751589, TimestampMs: 1723791285000}, + //{Value: 91293.822708, TimestampMs: 1723791300000}, + //{Value: 91303.847491, TimestampMs: 1723791315000}, + //{Value: 91323.330187, TimestampMs: 1723791330000}, + //{Value: 91333.262969, TimestampMs: 1723791345000}, + //{Value: 91344.740628, TimestampMs: 1723791360000}, + //{Value: 91354.903372, TimestampMs: 1723791375000}, + //{Value: 91367.906703, TimestampMs: 1723791390000}, + //{Value: 91378.553063, TimestampMs: 1723791405000}, + //{Value: 91394.636486, TimestampMs: 1723791420000}, + //{Value: 91410.204225, TimestampMs: 1723791435000}, + //{Value: 91422.493961, TimestampMs: 1723791450000}, + //{Value: 91433.819912, TimestampMs: 1723791465000}, + //{Value: 91443.898819, TimestampMs: 1723791480000}, + //{Value: 91459.981929, TimestampMs: 1723791495000}, + //{Value: 91470.091996, TimestampMs: 1723791510000}, + //{Value: 91486.074301, TimestampMs: 1723791525000}, + //{Value: 91500.000259, TimestampMs: 1723791540000}, + //{Value: 91500.000259, TimestampMs: 1723791555000}, + //{Value: 91525.132208, TimestampMs: 1723791570000}, + //{Value: 91538.572387, TimestampMs: 1723791585000}, + //{Value: 91552.01917, TimestampMs: 1723791600000}, + //{Value: 91563.870545, TimestampMs: 1723791615000}, + //{Value: 91574.424943, TimestampMs: 1723791630000}, + //{Value: 91589.326142, TimestampMs: 1723791645000}, + //{Value: 91599.479193, TimestampMs: 1723791660000}, + //{Value: 91616.369025, TimestampMs: 1723791675000}, + //{Value: 91616.369025, TimestampMs: 1723791690000}, + //{Value: 91641.707954, TimestampMs: 1723791705000}, + //{Value: 91650.981637, TimestampMs: 1723791720000}, + //{Value: 91663.434798, TimestampMs: 1723791735000}, + //{Value: 91679.998225, TimestampMs: 1723791750000}, + //{Value: 91679.998225, TimestampMs: 1723791765000}, + //{Value: 91703.713244, TimestampMs: 1723791780000}, + //{Value: 91715.546001, TimestampMs: 1723791795000}, + //{Value: 91715.546001, TimestampMs: 1723791810000}, + //{Value: 91732.134697, TimestampMs: 1723791825000}, + //{Value: 91747.85583, TimestampMs: 1723791840000}, + //{Value: 91759.740987, TimestampMs: 1723791855000}, + //{Value: 91771.015463, TimestampMs: 1723791870000}, + //{Value: 91784.90931, TimestampMs: 1723791885000}, + //{Value: 91800.034562, TimestampMs: 1723791900000}, + //{Value: 91813.36678, TimestampMs: 1723791915000}, + //{Value: 91826.134639, TimestampMs: 1723791930000}, + //{Value: 91840.4055, TimestampMs: 1723791945000}, + //{Value: 91850.050692, TimestampMs: 1723791960000}, + //{Value: 91864.529685, TimestampMs: 1723791975000}, + //{Value: 91879.74966, TimestampMs: 1723791990000}, + //{Value: 91896.755603, TimestampMs: 1723792005000}, + //{Value: 91896.755603, TimestampMs: 1723792020000}, + //{Value: 91912.51985, TimestampMs: 1723792035000}, + //{Value: 91933.378418, TimestampMs: 1723792050000}, + //{Value: 91947.25425, TimestampMs: 1723792065000}, + //{Value: 91960.974708, TimestampMs: 1723792080000}, + //{Value: 91970.51603, TimestampMs: 1723792095000}, + //{Value: 91980.078045, TimestampMs: 1723792110000}, + //{Value: 91995.510175, TimestampMs: 1723792125000}, + //{Value: 92008.588068, TimestampMs: 1723792140000}, + //{Value: 92022.377881, TimestampMs: 1723792155000}, + //{Value: 92032.231931, TimestampMs: 1723792170000}, + //{Value: 92045.974103, TimestampMs: 1723792185000}, + //{Value: 92057.361389, TimestampMs: 1723792200000}, + //{Value: 92073.260322, TimestampMs: 1723792215000}, + //{Value: 92082.037532, TimestampMs: 1723792230000}, + //{Value: 92095.530672, TimestampMs: 1723792245000}, + //{Value: 92107.013675, TimestampMs: 1723792260000}, + //{Value: 92119.510858, TimestampMs: 1723792275000}, + //{Value: 92132.906524, TimestampMs: 1723792290000}, + //{Value: 92145.149405, TimestampMs: 1723792305000}, + //{Value: 92160.632786, TimestampMs: 1723792320000}, + //{Value: 92171.862029, TimestampMs: 1723792335000}, + //{Value: 92186.777817, TimestampMs: 1723792350000}, + //{Value: 92199.992426, TimestampMs: 1723792365000}, + //{Value: 92215.714791, TimestampMs: 1723792380000}, + //{Value: 92215.714791, TimestampMs: 1723792395000}, + //{Value: 92231.82981, TimestampMs: 1723792410000}, + //{Value: 92244.71939, TimestampMs: 1723792425000}, + //{Value: 92258.157934, TimestampMs: 1723792440000}, + //{Value: 92276.828395, TimestampMs: 1723792455000}, + //{Value: 92288.292571, TimestampMs: 1723792470000}, + //{Value: 92288.292571, TimestampMs: 1723792485000}, + + // 2024-08-16 12:14:19 + {Value: 83110.230240566, TimestampMs: 1723781659695}, + // 2024-08-16 12:14:31 + {Value: 83119.89771368, TimestampMs: 1723781671058}, + // 14 + {Value: 83133.668603386, TimestampMs: 1723781687260}, + // + {Value: 83147.653484962, TimestampMs: 1723781703717}, + {Value: 83157.071587397, TimestampMs: 1723781714799}, + {Value: 83171.669781249, TimestampMs: 1723781731964}, + {Value: 83181.811100567, TimestampMs: 1723781743907}, + {Value: 83202.702299213, TimestampMs: 1723781768507}, + {Value: 83218.092858449, TimestampMs: 1723781786611}, + {Value: 83229.663421875, TimestampMs: 1723781800221}, + {Value: 83239.297822156, TimestampMs: 1723781811557}, + {Value: 83248.615420503, TimestampMs: 1723781822522}, + {Value: 83259.889640533, TimestampMs: 1723781835794}, + {Value: 83275.307092186, TimestampMs: 1723781853931}, + {Value: 83288.462744969, TimestampMs: 1723781869409}, + {Value: 83305.330399466, TimestampMs: 1723781889250}, + {Value: 83322.062551678, TimestampMs: 1723781908944}, + {Value: 83322.062551678, TimestampMs: 1723781908944}, + {Value: 83348.089331753, TimestampMs: 1723781939566}, + {Value: 83360.367689322, TimestampMs: 1723781954012}, + {Value: 83371.169228288, TimestampMs: 1723781966718}, + {Value: 83381.452888248, TimestampMs: 1723781978819}, + {Value: 83391.218637505, TimestampMs: 1723781990309}, + {Value: 83413.079859323, TimestampMs: 1723782016025}, + {Value: 83422.446287501, TimestampMs: 1723782027045}, + {Value: 83438.201603583, TimestampMs: 1723782045595}, + {Value: 83449.84403652, TimestampMs: 1723782059280}, + {Value: 83461.737374577, TimestampMs: 1723782073268}, + {Value: 83471.474445544, TimestampMs: 1723782084742}, + {Value: 83481.967704588, TimestampMs: 1723782097091}, + {Value: 83502.994504377, TimestampMs: 1723782121831}, + {Value: 83517.097210706, TimestampMs: 1723782138417}, + {Value: 83517.097210706, TimestampMs: 1723782138417}, + {Value: 83533.634034338, TimestampMs: 1723782157868}, + {Value: 83557.936638677, TimestampMs: 1723782186471}, + {Value: 83567.707069277, TimestampMs: 1723782197968}, + {Value: 83579.86422778, TimestampMs: 1723782212285}, + {Value: 83590.57310578, TimestampMs: 1723782224886}, + {Value: 83599.637284459, TimestampMs: 1723782235547}, + {Value: 83611.89931635, TimestampMs: 1723782249968}, + {Value: 83636.249048845, TimestampMs: 1723782278627}, + {Value: 83636.249048845, TimestampMs: 1723782278627}, + {Value: 83651.360371096, TimestampMs: 1723782296419}, + {Value: 83667.688770921, TimestampMs: 1723782315626}, + {Value: 83682.595590758, TimestampMs: 1723782333163}, + {Value: 83697.930974282, TimestampMs: 1723782351207}, + {Value: 83713.996785709, TimestampMs: 1723782370108}, + {Value: 83726.730892809, TimestampMs: 1723782385088}, + {Value: 83740.495341398, TimestampMs: 1723782401294}, + {Value: 83752.122844543, TimestampMs: 1723782414970}, + {Value: 83764.467953563, TimestampMs: 1723782429501}, + {Value: 83774.025976015, TimestampMs: 1723782440750}, + {Value: 83784.08183661, TimestampMs: 1723782452579}, + {Value: 83798.448884967, TimestampMs: 1723782469483}, + {Value: 83809.829149871, TimestampMs: 1723782482873}, + {Value: 83824.122434913, TimestampMs: 1723782499699}, + {Value: 83839.23376819, TimestampMs: 1723782517473}, + {Value: 83851.435562208, TimestampMs: 1723782531827}, + {Value: 83865.753165636, TimestampMs: 1723782548674}, + {Value: 83881.084374015, TimestampMs: 1723782566911}, + {Value: 83894.489397571, TimestampMs: 1723782582674}, + {Value: 83894.489397571, TimestampMs: 1723782582674}, + {Value: 83920.336955168, TimestampMs: 1723782613091}, + {Value: 83935.777101942, TimestampMs: 1723782631247}, + {Value: 83944.958825087, TimestampMs: 1723782642049}, + {Value: 83954.930618518, TimestampMs: 1723782653796}, + {Value: 83964.61676056, TimestampMs: 1723782665176}, + {Value: 83976.311339758, TimestampMs: 1723782678940}, + {Value: 83992.240992968, TimestampMs: 1723782697684}, + {Value: 84005.192565515, TimestampMs: 1723782712920}, + {Value: 84018.50751575, TimestampMs: 1723782728590}, + {Value: 84031.419153556, TimestampMs: 1723782743789}, + {Value: 84044.052355138, TimestampMs: 1723782758649}, + {Value: 84066.50921023, TimestampMs: 1723782785064}, + {Value: 84076.887606491, TimestampMs: 1723782797286}, + {Value: 84088.660599813, TimestampMs: 1723782811139}, + {Value: 84100.347848796, TimestampMs: 1723782824897}, + {Value: 84118.886448797, TimestampMs: 1723782846704}, + {Value: 84118.886448797, TimestampMs: 1723782846704}, + {Value: 84145.303297995, TimestampMs: 1723782877795}, + {Value: 84157.022505363, TimestampMs: 1723782891569}, + {Value: 84171.570885322, TimestampMs: 1723782908697}, + {Value: 84171.570885322, TimestampMs: 1723782908697}, + {Value: 84185.029034896, TimestampMs: 1723782924536}, + {Value: 84200.795632308, TimestampMs: 1723782943074}, + {Value: 84217.338205034, TimestampMs: 1723782962543}, + {Value: 84232.566871791, TimestampMs: 1723782980478}, + {Value: 84247.952157671, TimestampMs: 1723782998589}, + {Value: 84259.687514364, TimestampMs: 1723783012395}, + {Value: 84270.870575426, TimestampMs: 1723783025544}, + {Value: 84283.327474379, TimestampMs: 1723783040211}, + {Value: 84292.834037796, TimestampMs: 1723783051384}, + {Value: 84307.01297336, TimestampMs: 1723783068060}, + {Value: 84321.282299361, TimestampMs: 1723783084865}, + {Value: 84335.160947557, TimestampMs: 1723783101198}, + {Value: 84349.886931705, TimestampMs: 1723783118519}, + {Value: 84365.812366229, TimestampMs: 1723783137248}, + {Value: 84379.187285421, TimestampMs: 1723783152998}, + {Value: 84379.187285421, TimestampMs: 1723783152998}, + {Value: 84395.868999243, TimestampMs: 1723783172620}, + {Value: 84408.729856768, TimestampMs: 1723783187749}, + {Value: 84423.244350773, TimestampMs: 1723783204825}, + {Value: 84436.261392845, TimestampMs: 1723783220144}, + {Value: 84452.379284541, TimestampMs: 1723783239108}, + {Value: 84464.15768923, TimestampMs: 1723783252962}, + {Value: 84473.640088229, TimestampMs: 1723783264123}, + {Value: 84496.70578817, TimestampMs: 1723783291251}, + {Value: 84505.274005051, TimestampMs: 1723783301336}, + {Value: 84513.825175274, TimestampMs: 1723783311406}, + {Value: 84535.988590779, TimestampMs: 1723783337476}, + {Value: 84549.278094191, TimestampMs: 1723783353113}, + {Value: 84559.065807903, TimestampMs: 1723783364626}, + {Value: 84575.328212243, TimestampMs: 1723783383755}, + {Value: 84585.130274946, TimestampMs: 1723783395304}, + {Value: 84597.818248467, TimestampMs: 1723783410232}, + {Value: 84608.916253026, TimestampMs: 1723783423291}, + {Value: 84623.16625411, TimestampMs: 1723783440053}, + {Value: 84631.944909125, TimestampMs: 1723783450378}, + {Value: 84647.499217182, TimestampMs: 1723783468687}, + {Value: 84660.506692489, TimestampMs: 1723783484006}, + {Value: 84674.60893123, TimestampMs: 1723783500583}, + {Value: 84684.505685069, TimestampMs: 1723783512235}, + {Value: 84705.219658059, TimestampMs: 1723783536607}, + {Value: 84714.060915456, TimestampMs: 1723783547009}, + {Value: 84728.836124879, TimestampMs: 1723783564397}, + {Value: 84744.277426466, TimestampMs: 1723783582557}, + {Value: 84754.042469269, TimestampMs: 1723783594040}, + {Value: 84763.213702676, TimestampMs: 1723783604841}, + {Value: 84777.274878899, TimestampMs: 1723783621373}, + {Value: 84798.384659124, TimestampMs: 1723783646224}, + {Value: 84811.581806096, TimestampMs: 1723783661741}, + {Value: 84811.581806096, TimestampMs: 1723783661741}, + {Value: 84837.063572746, TimestampMs: 1723783691729}, + {Value: 84837.063572746, TimestampMs: 1723783691729}, + {Value: 84862.534809174, TimestampMs: 1723783721702}, + {Value: 84862.534809174, TimestampMs: 1723783721702}, + {Value: 84889.552986904, TimestampMs: 1723783753474}, + {Value: 84889.552986904, TimestampMs: 1723783753474}, + {Value: 84904.59475183, TimestampMs: 1723783771171}, + {Value: 84919.763932416, TimestampMs: 1723783789025}, + {Value: 84930.610916691, TimestampMs: 1723783801792}, + {Value: 84942.863905387, TimestampMs: 1723783816203}, + {Value: 84966.699414609, TimestampMs: 1723783844241}, + {Value: 84977.086454212, TimestampMs: 1723783856458}, + {Value: 84986.667846844, TimestampMs: 1723783867734}, + {Value: 85002.292679538, TimestampMs: 1723783886117}, + {Value: 85011.128037669, TimestampMs: 1723783896512}, + {Value: 85021.452751276, TimestampMs: 1723783908656}, + {Value: 85046.656737888, TimestampMs: 1723783938316}, + {Value: 85057.149303109, TimestampMs: 1723783950657}, + {Value: 85072.792663737, TimestampMs: 1723783969061}, + {Value: 85081.322519298, TimestampMs: 1723783979106}, + {Value: 85098.016534644, TimestampMs: 1723783998744}, + {Value: 85108.933812104, TimestampMs: 1723784011582}, + {Value: 85124.353810677, TimestampMs: 1723784029730}, + {Value: 85138.578992055, TimestampMs: 1723784046459}, + {Value: 85149.470757779, TimestampMs: 1723784059272}, + {Value: 85159.861953756, TimestampMs: 1723784071513}, + {Value: 85174.736633926, TimestampMs: 1723784089017}, + {Value: 85184.332789852, TimestampMs: 1723784100309}, + {Value: 85197.244814493, TimestampMs: 1723784115500}, + {Value: 85212.897341606, TimestampMs: 1723784133914}, + {Value: 85223.068417736, TimestampMs: 1723784145868}, + {Value: 85233.018931946, TimestampMs: 1723784157589}, + {Value: 85248.461401113, TimestampMs: 1723784175749}, + {Value: 85259.273556646, TimestampMs: 1723784188478}, + {Value: 85271.303120745, TimestampMs: 1723784202637}, + {Value: 85291.400852908, TimestampMs: 1723784226274}, + {Value: 85304.31033708, TimestampMs: 1723784241464}, + {Value: 85318.279367583, TimestampMs: 1723784257908}, + {Value: 85333.847398315, TimestampMs: 1723784276227}, + {Value: 85333.847398315, TimestampMs: 1723784276227}, + {Value: 85359.864088454, TimestampMs: 1723784306834}, + {Value: 85369.000247364, TimestampMs: 1723784317593}, + {Value: 85377.988741041, TimestampMs: 1723784328153}, + {Value: 85390.953853559, TimestampMs: 1723784343418}, + {Value: 85400.274651041, TimestampMs: 1723784354373}, + {Value: 85414.367124498, TimestampMs: 1723784370992}, + {Value: 85428.392179361, TimestampMs: 1723784387486}, + {Value: 85441.068542796, TimestampMs: 1723784402406}, + {Value: 85455.090792166, TimestampMs: 1723784418899}, + + {Value: 85469.916951658, TimestampMs: 1723784436340}, + // 85481.057819604-85469.916951658 = 11.140867946 + {Value: 85481.057819604, TimestampMs: 1723784449448}, + // 85492.957819034-85481.057819604 = 11.89999943 + {Value: 85492.957819034, TimestampMs: 1723784463447}, + // +34.5613010639936 @[1723784479695] (85492.957819034-85469.916951658)/(1723784463447-1723784436340)*1000*60 = 50.9998171 + // 23.040867376 / 27107 * 1000 * 60 + + {Value: 85507.169793341, TimestampMs: 1723784480165}, + {Value: 85523.205104841, TimestampMs: 1723784499037}, + {Value: 85534.871301536, TimestampMs: 1723784512760}, + {Value: 85550.176032782, TimestampMs: 1723784530770}, + ////// 50.99050225194026 @[1723784539695] (85550.176032782-85507.169793341)/(1723784530770-1723784480165)*1000*60 = 50.9905023 + //// 43.006239441 / 50605 * 1000 * 60 + // + //// + {Value: 85567.054311861, TimestampMs: 1723784550635}, + {Value: 85578.065671268, TimestampMs: 1723784563577}, + {Value: 85593.712675185, TimestampMs: 1723784581999}, + {Value: 85608.3864412, TimestampMs: 1723784599280}, + //// 50.980116360164764 @[1723784599695] (85608.3864412-85567.054311861)/(1723784599280-1723784550635)*1000*60 = 50.9801164 + // + //// + {Value: 85621.216462262, TimestampMs: 1723784614371}, + {Value: 85631.684176047, TimestampMs: 1723784626702}, + {Value: 85644.681704276, TimestampMs: 1723784641979}, + {Value: 85660.264511262, TimestampMs: 1723784660320}, + {Value: 85670.48807541, TimestampMs: 1723784672345}, + {Value: 85681.373292269, TimestampMs: 1723784685162}, + {Value: 85696.401263333, TimestampMs: 1723784702839}, + {Value: 85706.099345088, TimestampMs: 1723784714247}, + {Value: 85716.135249545, TimestampMs: 1723784726058}, + }, + }, + } + tsdb.SetStorage(defaultStorageID, &tsdb.Storage{ Instance: &instance{ name: defaultMetric, @@ -839,6 +1305,18 @@ NaN @[360000]`, 0 @[600000] 0 @[660000]`, }, + "test": { + q: fmt.Sprintf(`rate(%s[1m])`, defaultMetric), + start: time.UnixMilli(1723781659695), + end: time.UnixMilli(1723784726058), + step: time.Minute, + }, + "now": { + q: fmt.Sprintf(`rate(%s[1m])`, defaultMetric), + start: time.Unix(1723788909, 0), + end: time.Unix(1723792509, 0), + step: time.Minute, + }, } for k, c := range testCases { @@ -855,3 +1333,476 @@ NaN @[360000]`, } } + +func TestDemoGo(t *testing.T) { + a := []*remote.Sample{ + //{Value: 89254.443482, TimestampMs: 1723788900000}, + //{Value: 89271.419844, TimestampMs: 1723788915000}, + //{Value: 89281.17737, TimestampMs: 1723788930000}, + //{Value: 89294.234335, TimestampMs: 1723788945000}, + //{Value: 89310.166, TimestampMs: 1723788960000}, + //{Value: 89321.718902, TimestampMs: 1723788975000}, + //{Value: 89330.943656, TimestampMs: 1723788990000}, + //{Value: 89341.437103, TimestampMs: 1723789005000}, + //{Value: 89356.714587, TimestampMs: 1723789020000}, + //{Value: 89368.57525, TimestampMs: 1723789035000}, + //{Value: 89377.891269, TimestampMs: 1723789050000}, + //{Value: 89389.237067, TimestampMs: 1723789065000}, + //{Value: 89403.894449, TimestampMs: 1723789080000}, + //{Value: 89418.157712, TimestampMs: 1723789095000}, + //{Value: 89429.995274, TimestampMs: 1723789110000}, + //{Value: 89449.490131, TimestampMs: 1723789125000}, + //{Value: 89463.821452, TimestampMs: 1723789140000}, + //{Value: 89463.821452, TimestampMs: 1723789155000}, + //{Value: 89477.877679, TimestampMs: 1723789170000}, + //{Value: 89491.614274, TimestampMs: 1723789185000}, + //{Value: 89514.946665, TimestampMs: 1723789200000}, + //{Value: 89525.187888, TimestampMs: 1723789215000}, + //{Value: 89536.199943, TimestampMs: 1723789230000}, + //{Value: 89546.285583, TimestampMs: 1723789245000}, + //{Value: 89561.879647, TimestampMs: 1723789260000}, + //{Value: 89576.112585, TimestampMs: 1723789275000}, + //{Value: 89587.296897, TimestampMs: 1723789290000}, + //{Value: 89602.584535, TimestampMs: 1723789305000}, + //{Value: 89616.72048, TimestampMs: 1723789320000}, + //{Value: 89629.581803, TimestampMs: 1723789335000}, + //{Value: 89629.581803, TimestampMs: 1723789350000}, + //{Value: 89644.597443, TimestampMs: 1723789365000}, + //{Value: 89661.223251, TimestampMs: 1723789380000}, + //{Value: 89681.051627, TimestampMs: 1723789395000}, + //{Value: 89691.466575, TimestampMs: 1723789410000}, + //{Value: 89691.466575, TimestampMs: 1723789425000}, + //{Value: 89708.374236, TimestampMs: 1723789440000}, + //{Value: 89719.580165, TimestampMs: 1723789455000}, + //{Value: 89734.549785, TimestampMs: 1723789470000}, + //{Value: 89751.197662, TimestampMs: 1723789485000}, + //{Value: 89764.596974, TimestampMs: 1723789500000}, + //{Value: 89775.94415, TimestampMs: 1723789515000}, + //{Value: 89791.728723, TimestampMs: 1723789530000}, + //{Value: 89805.817496, TimestampMs: 1723789545000}, + //{Value: 89817.497365, TimestampMs: 1723789560000}, + //{Value: 89832.928259, TimestampMs: 1723789575000}, + //{Value: 89844.88877, TimestampMs: 1723789590000}, + //{Value: 89853.642183, TimestampMs: 1723789605000}, + //{Value: 89869.9064, TimestampMs: 1723789620000}, + //{Value: 89881.06497, TimestampMs: 1723789635000}, + //{Value: 89897.480595, TimestampMs: 1723789650000}, + //{Value: 89897.480595, TimestampMs: 1723789665000}, + //{Value: 89920.95489, TimestampMs: 1723789680000}, + //{Value: 89933.523817, TimestampMs: 1723789695000}, + //{Value: 89943.552657, TimestampMs: 1723789710000}, + //{Value: 89954.967603, TimestampMs: 1723789725000}, + //{Value: 89967.46319, TimestampMs: 1723789740000}, + //{Value: 89978.93852, TimestampMs: 1723789755000}, + //{Value: 89990.527473, TimestampMs: 1723789770000}, + //{Value: 90003.530246, TimestampMs: 1723789785000}, + //{Value: 90016.504908, TimestampMs: 1723789800000}, + //{Value: 90026.261807, TimestampMs: 1723789815000}, + //{Value: 90040.375899, TimestampMs: 1723789830000}, + //{Value: 90051.124329, TimestampMs: 1723789845000}, + //{Value: 90072.787219, TimestampMs: 1723789860000}, + //{Value: 90083.422473, TimestampMs: 1723789875000}, + //{Value: 90096.186985, TimestampMs: 1723789890000}, + //{Value: 90107.191318, TimestampMs: 1723789905000}, + //{Value: 90122.454399, TimestampMs: 1723789920000}, + //{Value: 90131.422952, TimestampMs: 1723789935000}, + //{Value: 90148.22782, TimestampMs: 1723789950000}, + //{Value: 90159.659718, TimestampMs: 1723789965000}, + //{Value: 90171.257627, TimestampMs: 1723789980000}, + //{Value: 90186.055175, TimestampMs: 1723789995000}, + //{Value: 90194.81363, TimestampMs: 1723790010000}, + //{Value: 90204.743511, TimestampMs: 1723790025000}, + //{Value: 90219.01107, TimestampMs: 1723790040000}, + //{Value: 90229.438247, TimestampMs: 1723790055000}, + //{Value: 90244.301206, TimestampMs: 1723790070000}, + //{Value: 90258.517053, TimestampMs: 1723790085000}, + //{Value: 90274.690343, TimestampMs: 1723790100000}, + //{Value: 90290.162218, TimestampMs: 1723790115000}, + //{Value: 90299.116268, TimestampMs: 1723790130000}, + //{Value: 90309.50359, TimestampMs: 1723790145000}, + //{Value: 90323.006066, TimestampMs: 1723790160000}, + //{Value: 90342.314282, TimestampMs: 1723790175000}, + //{Value: 90354.520428, TimestampMs: 1723790190000}, + //{Value: 90365.376755, TimestampMs: 1723790205000}, + //{Value: 90376.388391, TimestampMs: 1723790220000}, + //{Value: 90389.190165, TimestampMs: 1723790235000}, + //{Value: 90402.389841, TimestampMs: 1723790250000}, + //{Value: 90418.680096, TimestampMs: 1723790265000}, + //{Value: 90432.830444, TimestampMs: 1723790280000}, + //{Value: 90432.830444, TimestampMs: 1723790295000}, + //{Value: 90448.193001, TimestampMs: 1723790310000}, + //{Value: 90463.955856, TimestampMs: 1723790325000}, + //{Value: 90473.328555, TimestampMs: 1723790340000}, + //{Value: 90485.445743, TimestampMs: 1723790355000}, + //{Value: 90500.308975, TimestampMs: 1723790370000}, + //{Value: 90513.996363, TimestampMs: 1723790385000}, + //{Value: 90524.583692, TimestampMs: 1723790400000}, + //{Value: 90539.642466, TimestampMs: 1723790415000}, + //{Value: 90554.297539, TimestampMs: 1723790430000}, + //{Value: 90568.418917, TimestampMs: 1723790445000}, + //{Value: 90578.465894, TimestampMs: 1723790460000}, + //{Value: 90590.308212, TimestampMs: 1723790475000}, + //{Value: 90610.786895, TimestampMs: 1723790490000}, + //{Value: 90619.853563, TimestampMs: 1723790505000}, + //{Value: 90633.344401, TimestampMs: 1723790520000}, + //{Value: 90643.642528, TimestampMs: 1723790535000}, + //{Value: 90654.855284, TimestampMs: 1723790550000}, + //{Value: 90671.192098, TimestampMs: 1723790565000}, + //{Value: 90685.384962, TimestampMs: 1723790580000}, + //{Value: 90695.301196, TimestampMs: 1723790595000}, + //{Value: 90706.41933, TimestampMs: 1723790610000}, + //{Value: 90717.570439, TimestampMs: 1723790625000}, + //{Value: 90730.782064, TimestampMs: 1723790640000}, + //{Value: 90740.953431, TimestampMs: 1723790655000}, + //{Value: 90757.409502, TimestampMs: 1723790670000}, + //{Value: 90769.1984, TimestampMs: 1723790685000}, + //{Value: 90783.309249, TimestampMs: 1723790700000}, + //{Value: 90797.414003, TimestampMs: 1723790715000}, + //{Value: 90807.548935, TimestampMs: 1723790730000}, + //{Value: 90820.106166, TimestampMs: 1723790745000}, + //{Value: 90835.907679, TimestampMs: 1723790760000}, + //{Value: 90846.165194, TimestampMs: 1723790775000}, + //{Value: 90863.011773, TimestampMs: 1723790790000}, + //{Value: 90874.307508, TimestampMs: 1723790805000}, + //{Value: 90889.763538, TimestampMs: 1723790820000}, + //{Value: 90900.335085, TimestampMs: 1723790835000}, + //{Value: 90908.988402, TimestampMs: 1723790850000}, + //{Value: 90920.469334, TimestampMs: 1723790865000}, + //{Value: 90933.826482, TimestampMs: 1723790880000}, + //{Value: 90947.39323, TimestampMs: 1723790895000}, + //{Value: 90963.091511, TimestampMs: 1723790910000}, + //{Value: 90977.368179, TimestampMs: 1723790925000}, + //{Value: 90986.163197, TimestampMs: 1723790940000}, + //{Value: 90995.061955, TimestampMs: 1723790955000}, + //{Value: 91006.194755, TimestampMs: 1723790970000}, + //{Value: 91029.735577, TimestampMs: 1723790985000}, + //{Value: 91029.735577, TimestampMs: 1723791000000}, + //{Value: 91044.818113, TimestampMs: 1723791015000}, + //{Value: 91059.875089, TimestampMs: 1723791030000}, + //{Value: 91080.865333, TimestampMs: 1723791045000}, + //{Value: 91090.649849, TimestampMs: 1723791060000}, + //{Value: 91104.369272, TimestampMs: 1723791075000}, + //{Value: 91117.172565, TimestampMs: 1723791090000}, + //{Value: 91131.69536, TimestampMs: 1723791105000}, + //{Value: 91131.69536, TimestampMs: 1723791120000}, + //{Value: 91148.506416, TimestampMs: 1723791135000}, + //{Value: 91170.250174, TimestampMs: 1723791150000}, + //{Value: 91180.622433, TimestampMs: 1723791165000}, + //{Value: 91193.951386, TimestampMs: 1723791180000}, + //{Value: 91203.445343, TimestampMs: 1723791195000}, + //{Value: 91216.939029, TimestampMs: 1723791210000}, + //{Value: 91225.686877, TimestampMs: 1723791225000}, + //{Value: 91241.017744, TimestampMs: 1723791240000}, + //{Value: 91250.610082, TimestampMs: 1723791255000}, + //{Value: 91266.805103, TimestampMs: 1723791270000}, + //{Value: 91280.751589, TimestampMs: 1723791285000}, + //{Value: 91293.822708, TimestampMs: 1723791300000}, + //{Value: 91303.847491, TimestampMs: 1723791315000}, + //{Value: 91323.330187, TimestampMs: 1723791330000}, + //{Value: 91333.262969, TimestampMs: 1723791345000}, + //{Value: 91344.740628, TimestampMs: 1723791360000}, + //{Value: 91354.903372, TimestampMs: 1723791375000}, + //{Value: 91367.906703, TimestampMs: 1723791390000}, + //{Value: 91378.553063, TimestampMs: 1723791405000}, + //{Value: 91394.636486, TimestampMs: 1723791420000}, + //{Value: 91410.204225, TimestampMs: 1723791435000}, + //{Value: 91422.493961, TimestampMs: 1723791450000}, + //{Value: 91433.819912, TimestampMs: 1723791465000}, + //{Value: 91443.898819, TimestampMs: 1723791480000}, + //{Value: 91459.981929, TimestampMs: 1723791495000}, + //{Value: 91470.091996, TimestampMs: 1723791510000}, + //{Value: 91486.074301, TimestampMs: 1723791525000}, + //{Value: 91500.000259, TimestampMs: 1723791540000}, + //{Value: 91500.000259, TimestampMs: 1723791555000}, + //{Value: 91525.132208, TimestampMs: 1723791570000}, + //{Value: 91538.572387, TimestampMs: 1723791585000}, + //{Value: 91552.01917, TimestampMs: 1723791600000}, + //{Value: 91563.870545, TimestampMs: 1723791615000}, + //{Value: 91574.424943, TimestampMs: 1723791630000}, + //{Value: 91589.326142, TimestampMs: 1723791645000}, + //{Value: 91599.479193, TimestampMs: 1723791660000}, + //{Value: 91616.369025, TimestampMs: 1723791675000}, + //{Value: 91616.369025, TimestampMs: 1723791690000}, + //{Value: 91641.707954, TimestampMs: 1723791705000}, + //{Value: 91650.981637, TimestampMs: 1723791720000}, + //{Value: 91663.434798, TimestampMs: 1723791735000}, + //{Value: 91679.998225, TimestampMs: 1723791750000}, + //{Value: 91679.998225, TimestampMs: 1723791765000}, + //{Value: 91703.713244, TimestampMs: 1723791780000}, + //{Value: 91715.546001, TimestampMs: 1723791795000}, + //{Value: 91715.546001, TimestampMs: 1723791810000}, + //{Value: 91732.134697, TimestampMs: 1723791825000}, + //{Value: 91747.85583, TimestampMs: 1723791840000}, + //{Value: 91759.740987, TimestampMs: 1723791855000}, + //{Value: 91771.015463, TimestampMs: 1723791870000}, + //{Value: 91784.90931, TimestampMs: 1723791885000}, + //{Value: 91800.034562, TimestampMs: 1723791900000}, + //{Value: 91813.36678, TimestampMs: 1723791915000}, + //{Value: 91826.134639, TimestampMs: 1723791930000}, + //{Value: 91840.4055, TimestampMs: 1723791945000}, + //{Value: 91850.050692, TimestampMs: 1723791960000}, + //{Value: 91864.529685, TimestampMs: 1723791975000}, + //{Value: 91879.74966, TimestampMs: 1723791990000}, + //{Value: 91896.755603, TimestampMs: 1723792005000}, + //{Value: 91896.755603, TimestampMs: 1723792020000}, + //{Value: 91912.51985, TimestampMs: 1723792035000}, + //{Value: 91933.378418, TimestampMs: 1723792050000}, + //{Value: 91947.25425, TimestampMs: 1723792065000}, + //{Value: 91960.974708, TimestampMs: 1723792080000}, + //{Value: 91970.51603, TimestampMs: 1723792095000}, + //{Value: 91980.078045, TimestampMs: 1723792110000}, + //{Value: 91995.510175, TimestampMs: 1723792125000}, + //{Value: 92008.588068, TimestampMs: 1723792140000}, + //{Value: 92022.377881, TimestampMs: 1723792155000}, + //{Value: 92032.231931, TimestampMs: 1723792170000}, + //{Value: 92045.974103, TimestampMs: 1723792185000}, + //{Value: 92057.361389, TimestampMs: 1723792200000}, + //{Value: 92073.260322, TimestampMs: 1723792215000}, + //{Value: 92082.037532, TimestampMs: 1723792230000}, + //{Value: 92095.530672, TimestampMs: 1723792245000}, + //{Value: 92107.013675, TimestampMs: 1723792260000}, + //{Value: 92119.510858, TimestampMs: 1723792275000}, + //{Value: 92132.906524, TimestampMs: 1723792290000}, + //{Value: 92145.149405, TimestampMs: 1723792305000}, + //{Value: 92160.632786, TimestampMs: 1723792320000}, + //{Value: 92171.862029, TimestampMs: 1723792335000}, + //{Value: 92186.777817, TimestampMs: 1723792350000}, + //{Value: 92199.992426, TimestampMs: 1723792365000}, + //{Value: 92215.714791, TimestampMs: 1723792380000}, + //{Value: 92215.714791, TimestampMs: 1723792395000}, + //{Value: 92231.82981, TimestampMs: 1723792410000}, + //{Value: 92244.71939, TimestampMs: 1723792425000}, + //{Value: 92258.157934, TimestampMs: 1723792440000}, + //{Value: 92276.828395, TimestampMs: 1723792455000}, + //{Value: 92288.292571, TimestampMs: 1723792470000}, + //{Value: 92288.292571, TimestampMs: 1723792485000}, + + // 2024-08-16 12:14:19 + {Value: 83110.230240566, TimestampMs: 1723781659695}, + // 2024-08-16 12:14:31 + {Value: 83119.89771368, TimestampMs: 1723781671058}, + // 14 + {Value: 83133.668603386, TimestampMs: 1723781687260}, + // + {Value: 83147.653484962, TimestampMs: 1723781703717}, + {Value: 83157.071587397, TimestampMs: 1723781714799}, + {Value: 83171.669781249, TimestampMs: 1723781731964}, + {Value: 83181.811100567, TimestampMs: 1723781743907}, + {Value: 83202.702299213, TimestampMs: 1723781768507}, + {Value: 83218.092858449, TimestampMs: 1723781786611}, + {Value: 83229.663421875, TimestampMs: 1723781800221}, + {Value: 83239.297822156, TimestampMs: 1723781811557}, + {Value: 83248.615420503, TimestampMs: 1723781822522}, + {Value: 83259.889640533, TimestampMs: 1723781835794}, + {Value: 83275.307092186, TimestampMs: 1723781853931}, + {Value: 83288.462744969, TimestampMs: 1723781869409}, + {Value: 83305.330399466, TimestampMs: 1723781889250}, + {Value: 83322.062551678, TimestampMs: 1723781908944}, + {Value: 83322.062551678, TimestampMs: 1723781908944}, + {Value: 83348.089331753, TimestampMs: 1723781939566}, + {Value: 83360.367689322, TimestampMs: 1723781954012}, + {Value: 83371.169228288, TimestampMs: 1723781966718}, + {Value: 83381.452888248, TimestampMs: 1723781978819}, + {Value: 83391.218637505, TimestampMs: 1723781990309}, + {Value: 83413.079859323, TimestampMs: 1723782016025}, + {Value: 83422.446287501, TimestampMs: 1723782027045}, + {Value: 83438.201603583, TimestampMs: 1723782045595}, + {Value: 83449.84403652, TimestampMs: 1723782059280}, + {Value: 83461.737374577, TimestampMs: 1723782073268}, + {Value: 83471.474445544, TimestampMs: 1723782084742}, + {Value: 83481.967704588, TimestampMs: 1723782097091}, + {Value: 83502.994504377, TimestampMs: 1723782121831}, + {Value: 83517.097210706, TimestampMs: 1723782138417}, + {Value: 83517.097210706, TimestampMs: 1723782138417}, + {Value: 83533.634034338, TimestampMs: 1723782157868}, + {Value: 83557.936638677, TimestampMs: 1723782186471}, + {Value: 83567.707069277, TimestampMs: 1723782197968}, + {Value: 83579.86422778, TimestampMs: 1723782212285}, + {Value: 83590.57310578, TimestampMs: 1723782224886}, + {Value: 83599.637284459, TimestampMs: 1723782235547}, + {Value: 83611.89931635, TimestampMs: 1723782249968}, + {Value: 83636.249048845, TimestampMs: 1723782278627}, + {Value: 83636.249048845, TimestampMs: 1723782278627}, + {Value: 83651.360371096, TimestampMs: 1723782296419}, + {Value: 83667.688770921, TimestampMs: 1723782315626}, + {Value: 83682.595590758, TimestampMs: 1723782333163}, + {Value: 83697.930974282, TimestampMs: 1723782351207}, + {Value: 83713.996785709, TimestampMs: 1723782370108}, + {Value: 83726.730892809, TimestampMs: 1723782385088}, + {Value: 83740.495341398, TimestampMs: 1723782401294}, + {Value: 83752.122844543, TimestampMs: 1723782414970}, + {Value: 83764.467953563, TimestampMs: 1723782429501}, + {Value: 83774.025976015, TimestampMs: 1723782440750}, + {Value: 83784.08183661, TimestampMs: 1723782452579}, + {Value: 83798.448884967, TimestampMs: 1723782469483}, + {Value: 83809.829149871, TimestampMs: 1723782482873}, + {Value: 83824.122434913, TimestampMs: 1723782499699}, + {Value: 83839.23376819, TimestampMs: 1723782517473}, + {Value: 83851.435562208, TimestampMs: 1723782531827}, + {Value: 83865.753165636, TimestampMs: 1723782548674}, + {Value: 83881.084374015, TimestampMs: 1723782566911}, + {Value: 83894.489397571, TimestampMs: 1723782582674}, + {Value: 83894.489397571, TimestampMs: 1723782582674}, + {Value: 83920.336955168, TimestampMs: 1723782613091}, + {Value: 83935.777101942, TimestampMs: 1723782631247}, + {Value: 83944.958825087, TimestampMs: 1723782642049}, + {Value: 83954.930618518, TimestampMs: 1723782653796}, + {Value: 83964.61676056, TimestampMs: 1723782665176}, + {Value: 83976.311339758, TimestampMs: 1723782678940}, + {Value: 83992.240992968, TimestampMs: 1723782697684}, + {Value: 84005.192565515, TimestampMs: 1723782712920}, + {Value: 84018.50751575, TimestampMs: 1723782728590}, + {Value: 84031.419153556, TimestampMs: 1723782743789}, + {Value: 84044.052355138, TimestampMs: 1723782758649}, + {Value: 84066.50921023, TimestampMs: 1723782785064}, + {Value: 84076.887606491, TimestampMs: 1723782797286}, + {Value: 84088.660599813, TimestampMs: 1723782811139}, + {Value: 84100.347848796, TimestampMs: 1723782824897}, + {Value: 84118.886448797, TimestampMs: 1723782846704}, + {Value: 84118.886448797, TimestampMs: 1723782846704}, + {Value: 84145.303297995, TimestampMs: 1723782877795}, + {Value: 84157.022505363, TimestampMs: 1723782891569}, + {Value: 84171.570885322, TimestampMs: 1723782908697}, + {Value: 84171.570885322, TimestampMs: 1723782908697}, + {Value: 84185.029034896, TimestampMs: 1723782924536}, + {Value: 84200.795632308, TimestampMs: 1723782943074}, + {Value: 84217.338205034, TimestampMs: 1723782962543}, + {Value: 84232.566871791, TimestampMs: 1723782980478}, + {Value: 84247.952157671, TimestampMs: 1723782998589}, + {Value: 84259.687514364, TimestampMs: 1723783012395}, + {Value: 84270.870575426, TimestampMs: 1723783025544}, + {Value: 84283.327474379, TimestampMs: 1723783040211}, + {Value: 84292.834037796, TimestampMs: 1723783051384}, + {Value: 84307.01297336, TimestampMs: 1723783068060}, + {Value: 84321.282299361, TimestampMs: 1723783084865}, + {Value: 84335.160947557, TimestampMs: 1723783101198}, + {Value: 84349.886931705, TimestampMs: 1723783118519}, + {Value: 84365.812366229, TimestampMs: 1723783137248}, + {Value: 84379.187285421, TimestampMs: 1723783152998}, + {Value: 84379.187285421, TimestampMs: 1723783152998}, + {Value: 84395.868999243, TimestampMs: 1723783172620}, + {Value: 84408.729856768, TimestampMs: 1723783187749}, + {Value: 84423.244350773, TimestampMs: 1723783204825}, + {Value: 84436.261392845, TimestampMs: 1723783220144}, + {Value: 84452.379284541, TimestampMs: 1723783239108}, + {Value: 84464.15768923, TimestampMs: 1723783252962}, + {Value: 84473.640088229, TimestampMs: 1723783264123}, + {Value: 84496.70578817, TimestampMs: 1723783291251}, + {Value: 84505.274005051, TimestampMs: 1723783301336}, + {Value: 84513.825175274, TimestampMs: 1723783311406}, + {Value: 84535.988590779, TimestampMs: 1723783337476}, + {Value: 84549.278094191, TimestampMs: 1723783353113}, + {Value: 84559.065807903, TimestampMs: 1723783364626}, + {Value: 84575.328212243, TimestampMs: 1723783383755}, + {Value: 84585.130274946, TimestampMs: 1723783395304}, + {Value: 84597.818248467, TimestampMs: 1723783410232}, + {Value: 84608.916253026, TimestampMs: 1723783423291}, + {Value: 84623.16625411, TimestampMs: 1723783440053}, + {Value: 84631.944909125, TimestampMs: 1723783450378}, + {Value: 84647.499217182, TimestampMs: 1723783468687}, + {Value: 84660.506692489, TimestampMs: 1723783484006}, + {Value: 84674.60893123, TimestampMs: 1723783500583}, + {Value: 84684.505685069, TimestampMs: 1723783512235}, + {Value: 84705.219658059, TimestampMs: 1723783536607}, + {Value: 84714.060915456, TimestampMs: 1723783547009}, + {Value: 84728.836124879, TimestampMs: 1723783564397}, + {Value: 84744.277426466, TimestampMs: 1723783582557}, + {Value: 84754.042469269, TimestampMs: 1723783594040}, + {Value: 84763.213702676, TimestampMs: 1723783604841}, + {Value: 84777.274878899, TimestampMs: 1723783621373}, + {Value: 84798.384659124, TimestampMs: 1723783646224}, + {Value: 84811.581806096, TimestampMs: 1723783661741}, + {Value: 84811.581806096, TimestampMs: 1723783661741}, + {Value: 84837.063572746, TimestampMs: 1723783691729}, + {Value: 84837.063572746, TimestampMs: 1723783691729}, + {Value: 84862.534809174, TimestampMs: 1723783721702}, + {Value: 84862.534809174, TimestampMs: 1723783721702}, + {Value: 84889.552986904, TimestampMs: 1723783753474}, + {Value: 84889.552986904, TimestampMs: 1723783753474}, + {Value: 84904.59475183, TimestampMs: 1723783771171}, + {Value: 84919.763932416, TimestampMs: 1723783789025}, + {Value: 84930.610916691, TimestampMs: 1723783801792}, + {Value: 84942.863905387, TimestampMs: 1723783816203}, + {Value: 84966.699414609, TimestampMs: 1723783844241}, + {Value: 84977.086454212, TimestampMs: 1723783856458}, + {Value: 84986.667846844, TimestampMs: 1723783867734}, + {Value: 85002.292679538, TimestampMs: 1723783886117}, + {Value: 85011.128037669, TimestampMs: 1723783896512}, + {Value: 85021.452751276, TimestampMs: 1723783908656}, + {Value: 85046.656737888, TimestampMs: 1723783938316}, + {Value: 85057.149303109, TimestampMs: 1723783950657}, + {Value: 85072.792663737, TimestampMs: 1723783969061}, + {Value: 85081.322519298, TimestampMs: 1723783979106}, + {Value: 85098.016534644, TimestampMs: 1723783998744}, + {Value: 85108.933812104, TimestampMs: 1723784011582}, + {Value: 85124.353810677, TimestampMs: 1723784029730}, + {Value: 85138.578992055, TimestampMs: 1723784046459}, + {Value: 85149.470757779, TimestampMs: 1723784059272}, + {Value: 85159.861953756, TimestampMs: 1723784071513}, + {Value: 85174.736633926, TimestampMs: 1723784089017}, + {Value: 85184.332789852, TimestampMs: 1723784100309}, + {Value: 85197.244814493, TimestampMs: 1723784115500}, + {Value: 85212.897341606, TimestampMs: 1723784133914}, + {Value: 85223.068417736, TimestampMs: 1723784145868}, + {Value: 85233.018931946, TimestampMs: 1723784157589}, + {Value: 85248.461401113, TimestampMs: 1723784175749}, + {Value: 85259.273556646, TimestampMs: 1723784188478}, + {Value: 85271.303120745, TimestampMs: 1723784202637}, + {Value: 85291.400852908, TimestampMs: 1723784226274}, + {Value: 85304.31033708, TimestampMs: 1723784241464}, + {Value: 85318.279367583, TimestampMs: 1723784257908}, + {Value: 85333.847398315, TimestampMs: 1723784276227}, + {Value: 85333.847398315, TimestampMs: 1723784276227}, + {Value: 85359.864088454, TimestampMs: 1723784306834}, + {Value: 85369.000247364, TimestampMs: 1723784317593}, + {Value: 85377.988741041, TimestampMs: 1723784328153}, + {Value: 85390.953853559, TimestampMs: 1723784343418}, + {Value: 85400.274651041, TimestampMs: 1723784354373}, + {Value: 85414.367124498, TimestampMs: 1723784370992}, + {Value: 85428.392179361, TimestampMs: 1723784387486}, + {Value: 85441.068542796, TimestampMs: 1723784402406}, + {Value: 85455.090792166, TimestampMs: 1723784418899}, + + {Value: 85469.916951658, TimestampMs: 1723784436340}, + // 85481.057819604-85469.916951658 = 11.140867946 + {Value: 85481.057819604, TimestampMs: 1723784449448}, + // 85492.957819034-85481.057819604 = 11.89999943 + {Value: 85492.957819034, TimestampMs: 1723784463447}, + // +34.5613010639936 @[1723784479695] (85492.957819034-85469.916951658)/(1723784463447-1723784436340)*1000*60 = 50.9998171 + // 23.040867376 / 27107 * 1000 * 60 + + {Value: 85507.169793341, TimestampMs: 1723784480165}, + {Value: 85523.205104841, TimestampMs: 1723784499037}, + {Value: 85534.871301536, TimestampMs: 1723784512760}, + {Value: 85550.176032782, TimestampMs: 1723784530770}, + ////// 50.99050225194026 @[1723784539695] (85550.176032782-85507.169793341)/(1723784530770-1723784480165)*1000*60 = 50.9905023 + //// 43.006239441 / 50605 * 1000 * 60 + // + //// + {Value: 85567.054311861, TimestampMs: 1723784550635}, + {Value: 85578.065671268, TimestampMs: 1723784563577}, + {Value: 85593.712675185, TimestampMs: 1723784581999}, + {Value: 85608.3864412, TimestampMs: 1723784599280}, + //// 50.980116360164764 @[1723784599695] (85608.3864412-85567.054311861)/(1723784599280-1723784550635)*1000*60 = 50.9801164 + // + //// + {Value: 85621.216462262, TimestampMs: 1723784614371}, + {Value: 85631.684176047, TimestampMs: 1723784626702}, + {Value: 85644.681704276, TimestampMs: 1723784641979}, + {Value: 85660.264511262, TimestampMs: 1723784660320}, + {Value: 85670.48807541, TimestampMs: 1723784672345}, + {Value: 85681.373292269, TimestampMs: 1723784685162}, + {Value: 85696.401263333, TimestampMs: 1723784702839}, + {Value: 85706.099345088, TimestampMs: 1723784714247}, + {Value: 85716.135249545, TimestampMs: 1723784726058}, + } + + for i, v := range a { + if i > 0 { + last := a[i-1] + if last.TimestampMs != v.TimestampMs { + fmt.Printf("%d, value: %f / time: %d / increase 1m: %f\n", v.TimestampMs, v.Value-last.Value, v.TimestampMs-last.TimestampMs, (float64(v.Value-last.Value) * 1000 / float64(v.TimestampMs-last.TimestampMs))) + } + } + } +} diff --git a/pkg/unify-query/tsdb/prometheus/querier.go b/pkg/unify-query/tsdb/prometheus/querier.go index 6a4cfe50f..87927b797 100644 --- a/pkg/unify-query/tsdb/prometheus/querier.go +++ b/pkg/unify-query/tsdb/prometheus/querier.go @@ -20,15 +20,21 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/bkapi" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + baseInfluxdb "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" + tsDBService "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/service/tsdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" - tsDBInfluxdb "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/victoriaMetrics" + routerInfluxdb "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/router/influxdb" ) const ( @@ -86,7 +92,7 @@ func (q *Querier) getQueryList(referenceName string) []*Query { if queryMetric, ok := queries[referenceName]; ok { queryList = make([]*Query, 0, len(queryMetric.QueryList)) for _, qry := range queryMetric.QueryList { - instance := GetInstance(ctx, qry) + instance := GetTsDbInstance(ctx, qry) if instance != nil { queryList = append(queryList, &Query{ instance: instance, @@ -262,8 +268,8 @@ func (q *Querier) LabelValues(name string, matchers ...*labels.Matcher) ([]strin } metadata.SetExpand(ctx, vmExpand) - instance := GetInstance(ctx, &metadata.Query{ - StorageID: consul.VictoriaMetricsStorageType, + instance := GetTsDbInstance(ctx, &metadata.Query{ + StorageType: consul.VictoriaMetricsStorageType, }) if instance == nil { err = fmt.Errorf("%s storage get error", consul.VictoriaMetricsStorageType) @@ -329,8 +335,8 @@ func (q *Querier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.War } metadata.SetExpand(ctx, vmExpand) - instance := GetInstance(ctx, &metadata.Query{ - StorageID: consul.VictoriaMetricsStorageType, + instance := GetTsDbInstance(ctx, &metadata.Query{ + StorageType: consul.VictoriaMetricsStorageType, }) if instance == nil { err = fmt.Errorf("%s storage get error", consul.VictoriaMetricsStorageType) @@ -372,86 +378,116 @@ func (q *Querier) Close() error { return nil } -// GetInstance 通过 qry 获取实例 -func GetInstance(ctx context.Context, qry *metadata.Query) tsdb.Instance { +func GetTsDbInstance(ctx context.Context, qry *metadata.Query) tsdb.Instance { var ( instance tsdb.Instance err error ) - ctx, span := trace.NewSpan(ctx, "stg-get-instance") - defer span.End(&err) - stg, err := tsdb.GetStorage(qry.StorageID) - if err != nil { - log.Errorf( - ctx, "get stg error: %s.%s: %s", qry.DB, qry.Measurement, err.Error(), - ) - return nil - } - if stg.Instance != nil { - return stg.Instance - } - span.Set("stroage-type", stg.Type) - span.Set("stg-id", qry.StorageID) - span.Set("stg-address", stg.Address) - span.Set("stg-uri-path", stg.UriPath) - - curlGet := &curl.HttpCurl{Log: log.DefaultLogger} - switch stg.Type { - // vm 实例直接在 stg.instance 就有了，无需进到这个逻辑 - case consul.ElasticsearchStorageType: - instOption := &elasticsearch.InstanceOption{ - Address: stg.Address, - Username: stg.Username, - Password: stg.Password, - MaxSize: stg.MaxLimit, - Timeout: stg.Timeout, - MaxRouting: stg.MaxRouting, - } - instance, err = elasticsearch.NewInstance(ctx, instOption) + ctx, span := trace.NewSpan(ctx, "get-ts-db-instance") + defer func() { if err != nil { log.Errorf(ctx, err.Error()) - return nil } + span.End(&err) + }() + + span.Set("storage-id", qry.StorageID) + + // 兼容原逻辑，storageType 通过 storageMap 获取 + stg, err := tsdb.GetStorage(qry.StorageID) + if stg != nil { + qry.StorageType = stg.Type + } + + span.Set("storage-type", qry.StorageType) + curlGet := &curl.HttpCurl{Log: log.DefaultLogger} + + switch qry.StorageType { case consul.InfluxDBStorageType: - insOption := tsDBInfluxdb.Options{ - ReadRateLimit: stg.ReadRateLimit, - Timeout: stg.Timeout, - ContentType: stg.ContentType, - ChunkSize: stg.ChunkSize, - RawUriPath: stg.UriPath, - Accept: stg.Accept, - AcceptEncoding: stg.AcceptEncoding, - MaxLimit: stg.MaxLimit, - MaxSlimit: stg.MaxSLimit, - Tolerance: stg.Toleration, + opt := influxdb.Options{ + Timeout: tsDBService.InfluxDBTimeout, + ContentType: tsDBService.InfluxDBContentType, + ChunkSize: tsDBService.InfluxDBChunkSize, + RawUriPath: tsDBService.InfluxDBQueryRawUriPath, + Accept: tsDBService.InfluxDBQueryRawAccept, + AcceptEncoding: tsDBService.InfluxDBQueryRawAcceptEncoding, + MaxLimit: tsDBService.InfluxDBMaxLimit, + MaxSlimit: tsDBService.InfluxDBMaxSLimit, + Tolerance: tsDBService.InfluxDBTolerance, + ReadRateLimit: tsDBService.InfluxDBQueryReadRateLimit, Curl: curlGet, } - - host, err := influxdb.GetInfluxDBRouter().GetInfluxDBHost( + var host *routerInfluxdb.Host + host, err = baseInfluxdb.GetInfluxDBRouter().GetInfluxDBHost( ctx, qry.TagsKey, qry.ClusterName, qry.DB, qry.Measurement, qry.Condition, ) if err != nil { - log.Errorf(ctx, err.Error()) return nil } - insOption.Host = host.DomainName - insOption.Port = host.Port - insOption.GrpcPort = host.GrpcPort - insOption.Protocol = host.Protocol - insOption.Username = host.Username - insOption.Password = host.Password - + opt.Host = host.DomainName + opt.Port = host.Port + opt.GrpcPort = host.GrpcPort + opt.Protocol = host.Protocol + opt.Username = host.Username + opt.Password = host.Password // 如果 host 有单独配置，则替换默认限速配置 if host.ReadRateLimit > 0 { - insOption.ReadRateLimit = host.ReadRateLimit + opt.ReadRateLimit = host.ReadRateLimit } - instance = tsDBInfluxdb.NewInstance(ctx, insOption) span.Set("cluster-name", qry.ClusterName) span.Set("tag-keys", fmt.Sprintf("%+v", qry.TagsKey)) - span.Set("ins-option", fmt.Sprintf("%+v", insOption)) + span.Set("ins-option", fmt.Sprintf("%+v", opt)) + + instance, err = influxdb.NewInstance(ctx, opt) + case consul.ElasticsearchStorageType: + opt := &elasticsearch.InstanceOption{ + MaxSize: tsDBService.EsMaxSize, + Timeout: tsDBService.EsTimeout, + MaxRouting: tsDBService.EsMaxRouting, + } + if qry.SourceType == structured.BkData { + opt.Address = bkapi.GetBkDataApi().QueryEsUrl() + opt.Headers = bkapi.GetBkDataApi().HttpHeaders(nil) + opt.HealthCheck = false + } else { + if stg == nil { + err = fmt.Errorf("%s storage is nil in %s", consul.ElasticsearchStorageType, qry.StorageID) + return nil + } + opt.Address = stg.Address + opt.Username = stg.Username + opt.Password = stg.Password + opt.HealthCheck = true + } + instance, err = elasticsearch.NewInstance(ctx, opt) + case consul.BkSqlStorageType: + instance, err = bksql.NewInstance(ctx, bksql.Options{ + Address: bkapi.GetBkDataApi().QuerySyncUrl(), + Headers: bkapi.GetBkDataApi().Headers(map[string]string{ + bksql.ContentType: tsDBService.BkSqlContentType, + }), + Timeout: tsDBService.BkSqlTimeout, + IntervalTime: tsDBService.BkSqlIntervalTime, + MaxLimit: tsDBService.BkSqlLimit, + Tolerance: tsDBService.BkSqlTolerance, + Curl: curlGet, + }) + case consul.VictoriaMetricsStorageType: + instance, err = victoriaMetrics.NewInstance(ctx, victoriaMetrics.Options{ + Address: bkapi.GetBkDataApi().QuerySyncUrl(), + Headers: bkapi.GetBkDataApi().Headers(map[string]string{ + victoriaMetrics.ContentType: tsDBService.VmContentType, + }), + MaxConditionNum: tsDBService.VmMaxConditionNum, + Timeout: tsDBService.VmTimeout, + InfluxCompatible: tsDBService.VmInfluxCompatible, + UseNativeOr: tsDBService.VmUseNativeOr, + Curl: curlGet, + }) default: + err = fmt.Errorf("sotrage type is error %+v", qry) return nil } diff --git a/pkg/unify-query/tsdb/storage.go b/pkg/unify-query/tsdb/storage.go index 6c4422e86..468490e3f 100644 --- a/pkg/unify-query/tsdb/storage.go +++ b/pkg/unify-query/tsdb/storage.go @@ -39,9 +39,6 @@ func ReloadTsDBStorage(_ context.Context, tsDBs map[string]*consul.Storage, opt } switch tsDB.Type { - case consul.VictoriaMetricsStorageType: - storage.UriPath = opt.VM.UriPath - storage.Timeout = opt.VM.Timeout case consul.ElasticsearchStorageType: storage.Timeout = opt.Es.Timeout storage.MaxRouting = opt.Es.MaxRouting diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance.go b/pkg/unify-query/tsdb/victoriaMetrics/instance.go index 11a3e7d81..d956035f5 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance.go @@ -49,47 +49,57 @@ const ( MatrixType = "matrix" ) -// Instance vm 查询实例 -type Instance struct { - Ctx context.Context - +type Options struct { + Address string + Headers map[string]string MaxConditionNum int + Timeout time.Duration + Curl curl.Curl - ContentType string + InfluxCompatible bool + UseNativeOr bool +} - Address string - UriPath string +// Instance vm 查询实例 +type Instance struct { + ctx context.Context - Code string - Secret string - Token string + maxConditionNum int - AuthenticationMethod string + url string + headers map[string]string - InfluxCompatible bool - UseNativeOr bool + influxCompatible bool + useNativeOr bool - Timeout time.Duration - Curl curl.Curl + timeout time.Duration + curl curl.Curl } var _ tsdb.Instance = (*Instance)(nil) +func NewInstance(ctx context.Context, opt Options) (*Instance, error) { + if opt.Address == "" { + return nil, fmt.Errorf("address is empty") + } + instance := &Instance{ + ctx: ctx, + maxConditionNum: opt.MaxConditionNum, + url: opt.Address, + headers: opt.Headers, + influxCompatible: opt.InfluxCompatible, + useNativeOr: opt.UseNativeOr, + timeout: opt.Timeout, + curl: opt.Curl, + } + return instance, nil +} + func (i *Instance) QueryRaw(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { //TODO implement me panic("implement me") } -func (i *Instance) authorization() string { - auth := fmt.Sprintf( - `{"bk_username": "%s", "bk_app_code": "%s", "bk_app_secret": "%s"}`, - BkUserName, - i.Code, - i.Secret, - ) - return auth -} - func (i *Instance) vectorFormat(ctx context.Context, resp *VmResponse, span *trace.Span) (promql.Vector, error) { if !resp.Result { return nil, fmt.Errorf( @@ -310,44 +320,40 @@ func (i *Instance) vmQuery( err error ) - address := fmt.Sprintf("%s/%s", i.Address, i.UriPath) user := metadata.GetUser(ctx) - params := &Params{ - SQL: sql, - BkdataAuthenticationMethod: i.AuthenticationMethod, - BkAppCode: i.Code, - PreferStorage: PreferStorage, - BkdataDataToken: i.Token, - } + + params := make(map[string]string) + params["sql"] = sql + params["prefer_storage"] = PreferStorage body, err := json.Marshal(params) if err != nil { return err } - ctx, cancel = context.WithTimeout(ctx, i.Timeout) + ctx, cancel = context.WithTimeout(ctx, i.timeout) defer cancel() startAnaylize = time.Now() span.Set("query-source", user.Source) span.Set("query-space-uid", user.SpaceUid) span.Set("query-username", user.Name) - span.Set("query-address", i.Address) - span.Set("query-uri-path", i.UriPath) + + span.Set("query-address", i.url) + + headersString, _ := json.Marshal(i.headers) + span.Set("query-headers", headersString) log.Infof(ctx, - "victoria metrics query: %s, body: %s, sql: %s", - address, body, sql, + "victoria metrics query: %s, headers: %s, body: %s", + i.url, headersString, body, ) - size, err := i.Curl.Request( + size, err := i.curl.Request( ctx, curl.Post, curl.Options{ - UrlPath: address, + UrlPath: i.url, Body: body, - Headers: map[string]string{ - ContentType: i.ContentType, - Authorization: i.authorization(), - }, + Headers: i.headers, }, data, ) @@ -396,12 +402,12 @@ func (i *Instance) QueryRange( ves, _ := json.Marshal(vmExpand) log.Infof(ctx, "vm-expand: %s", ves) - if i.MaxConditionNum > 0 && vmExpand.ConditionNum > i.MaxConditionNum { - return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.MaxConditionNum) + if i.maxConditionNum > 0 && vmExpand.ConditionNum > i.maxConditionNum { + return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.maxConditionNum) } paramsQueryRange := &ParamsQueryRange{ - InfluxCompatible: i.InfluxCompatible, + InfluxCompatible: i.influxCompatible, APIType: APIQueryRange, APIParams: struct { Query string `json:"query"` @@ -414,7 +420,7 @@ func (i *Instance) QueryRange( End: end.Unix(), Step: int64(step.Seconds()), }, - UseNativeOr: i.UseNativeOr, + UseNativeOr: i.useNativeOr, MetricFilterCondition: vmExpand.MetricFilterCondition, ResultTableList: vmExpand.ResultTableList, } @@ -463,12 +469,12 @@ func (i *Instance) Query( ves, _ := json.Marshal(vmExpand) span.Set("vm-expand", string(ves)) - if i.MaxConditionNum > 0 && vmExpand.ConditionNum > i.MaxConditionNum { - return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.MaxConditionNum) + if i.maxConditionNum > 0 && vmExpand.ConditionNum > i.maxConditionNum { + return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.maxConditionNum) } paramsQuery := &ParamsQuery{ - InfluxCompatible: i.InfluxCompatible, + InfluxCompatible: i.influxCompatible, APIType: APIQuery, APIParams: struct { Query string `json:"query"` @@ -477,9 +483,9 @@ func (i *Instance) Query( }{ Query: promqlStr, Time: end.Unix(), - Timeout: int64(i.Timeout.Seconds()), + Timeout: int64(i.timeout.Seconds()), }, - UseNativeOr: i.UseNativeOr, + UseNativeOr: i.useNativeOr, MetricFilterCondition: vmExpand.MetricFilterCondition, ResultTableList: vmExpand.ResultTableList, } @@ -520,22 +526,22 @@ func (i *Instance) metric(ctx context.Context, name string, matchers ...*labels. return nil, nil } - if i.MaxConditionNum > 0 && vmExpand.ConditionNum > i.MaxConditionNum { - return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.MaxConditionNum) + if i.maxConditionNum > 0 && vmExpand.ConditionNum > i.maxConditionNum { + return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.maxConditionNum) } ves, _ := json.Marshal(vmExpand) span.Set("vm-expand", string(ves)) paramsQuery := &ParamsLabelValues{ - InfluxCompatible: i.InfluxCompatible, + InfluxCompatible: i.influxCompatible, APIType: APILabelValues, APIParams: struct { Label string `json:"label"` }{ Label: name, }, - UseNativeOr: i.UseNativeOr, + UseNativeOr: i.useNativeOr, MetricFilterCondition: vmExpand.MetricFilterCondition, ResultTableList: vmExpand.ResultTableList, } @@ -577,8 +583,8 @@ func (i *Instance) LabelNames(ctx context.Context, query *metadata.Query, start, ves, _ := json.Marshal(vmExpand) span.Set("vm-expand", string(ves)) - if i.MaxConditionNum > 0 && vmExpand.ConditionNum > i.MaxConditionNum { - return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.MaxConditionNum) + if i.maxConditionNum > 0 && vmExpand.ConditionNum > i.maxConditionNum { + return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.maxConditionNum) } span.Set("query-matchers", fmt.Sprintf("%+v", matchers)) @@ -586,7 +592,7 @@ func (i *Instance) LabelNames(ctx context.Context, query *metadata.Query, start, span.Set("query-end", end.String()) paramsQuery := &ParamsSeries{ - InfluxCompatible: i.InfluxCompatible, + InfluxCompatible: i.influxCompatible, APIType: APILabelNames, APIParams: struct { Match string `json:"match[]"` @@ -596,7 +602,7 @@ func (i *Instance) LabelNames(ctx context.Context, query *metadata.Query, start, Start: start.Unix(), End: end.Unix(), }, - UseNativeOr: i.UseNativeOr, + UseNativeOr: i.useNativeOr, MetricFilterCondition: vmExpand.MetricFilterCondition, ResultTableList: vmExpand.ResultTableList, } @@ -645,8 +651,8 @@ func (i *Instance) LabelValues(ctx context.Context, query *metadata.Query, name return nil, nil } - if i.MaxConditionNum > 0 && vmExpand.ConditionNum > i.MaxConditionNum { - return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.MaxConditionNum) + if i.maxConditionNum > 0 && vmExpand.ConditionNum > i.maxConditionNum { + return nil, fmt.Errorf("condition length is too long %d > %d", vmExpand.ConditionNum, i.maxConditionNum) } ves, _ := json.Marshal(vmExpand) @@ -674,7 +680,7 @@ func (i *Instance) LabelValues(ctx context.Context, query *metadata.Query, name } paramsQueryRange := &ParamsQueryRange{ - InfluxCompatible: i.InfluxCompatible, + InfluxCompatible: i.influxCompatible, APIType: APIQueryRange, APIParams: struct { Query string `json:"query"` @@ -686,7 +692,7 @@ func (i *Instance) LabelValues(ctx context.Context, query *metadata.Query, name End: end.Unix(), Step: step, }, - UseNativeOr: i.UseNativeOr, + UseNativeOr: i.useNativeOr, MetricFilterCondition: vmExpand.MetricFilterCondition, ResultTableList: vmExpand.ResultTableList, } diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go index 5333329fd..7b30487cd 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go @@ -22,11 +22,9 @@ import ( "time" "github.com/prometheus/prometheus/model/labels" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/featureFlag" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" @@ -100,16 +98,10 @@ func TestPromQL(t *testing.T) { once.Do(func() { instance = &Instance{ - ContentType: "application/json", - Address: "http://127.0.0.1", - UriPath: "prod/v3/queryengine/query_sync", - Code: "bkmonitorv3", - Secret: "", - Token: "", - AuthenticationMethod: "token", - InfluxCompatible: true, - UseNativeOr: true, - Timeout: time.Second * 30, + ContentType: "application/json", + InfluxCompatible: true, + UseNativeOr: true, + Timeout: time.Second * 30, Curl: &curl.HttpCurl{ Log: log.DefaultLogger, }, @@ -180,32 +172,22 @@ func TestPromQL(t *testing.T) { func TestRealQueryRange(t *testing.T) { mock.Init() + ctx := metadata.InitHashID(context.Background()) - ctx := context.Background() - a := "a" - timeout := time.Minute + vmRT := "2_bcs_prom_computation_result_table" + metric := "container_cpu_usage_seconds_total" - flag := `{"vm-query-or":{"variations":{"vm":true,"influxdb":false},"defaultRule":{"percentage":{"vm":100,"influxdb":0}}}}` - featureFlag.MockFeatureFlag(ctx, flag) + a := "a" + metricFilterCondition := map[string]string{ + a: fmt.Sprintf(`__name__="%s_value", result_table_id="%s"`, metric, vmRT), + } - ctx = metadata.InitHashID(ctx) - address := viper.GetString("mock.victoria_metrics.address") - uriPath := viper.GetString("mock.victoria_metrics.uri_path") - code := viper.GetString("mock.victoria_metrics.code") - secret := viper.GetString("mock.victoria_metrics.secret") - token := viper.GetString("mock.victoria_metrics.token") - method := viper.GetString("mock.victoria_metrics.authentication_method") + timeout := time.Minute ins := &Instance{ - Ctx: ctx, - Address: address, - UriPath: uriPath, - Code: code, - Secret: secret, - AuthenticationMethod: method, - Timeout: timeout, - ContentType: "application/json", - Token: token, + ctx: ctx, + Timeout: timeout, + ContentType: "application/json", Curl: &curl.HttpCurl{Log: log.DefaultLogger}, @@ -221,12 +203,10 @@ func TestRealQueryRange(t *testing.T) { q: `count(a)`, e: &metadata.VmExpand{ ResultTableList: []string{ - "100147_bcs_custom_metric_result_table_40708", + vmRT, }, // condition 需要进行二次转义 - MetricFilterCondition: map[string]string{ - a: `__name__="envoy_listener_manager_worker_0_dispatcher_poll_delay_us_bucket_value", result_table_id="100147_bcs_custom_metric_result_table_40708"`, - }, + MetricFilterCondition: metricFilterCondition, }, }, } @@ -306,9 +286,8 @@ func TestInstance_Query_Url(t *testing.T) { ctx := metadata.InitHashID(context.Background()) ins := &Instance{ - Ctx: ctx, - Address: "http://127.0.0.1/api", - Curl: mockCurl, + ctx: ctx, + Curl: mockCurl, } mockData(ctx) @@ -361,8 +340,7 @@ func TestInstance_QueryRange_Url(t *testing.T) { }, log.DefaultLogger) ins := &Instance{ - Ctx: ctx, - Address: "http://127.0.0.1/api", + ctx: ctx, Timeout: time.Minute, Curl: mockCurl, } @@ -415,18 +393,12 @@ func TestInstance_QueryRange_Url(t *testing.T) { func mockInstance(ctx context.Context) { instance = &Instance{ - Ctx: ctx, - ContentType: "application/json", - Address: "http://127.0.0.1", - UriPath: "query_sync", - Code: "bkmonitorv3", - Secret: "", - Token: "", - AuthenticationMethod: "token", - InfluxCompatible: true, - UseNativeOr: true, - Timeout: time.Minute, - Curl: &curl.HttpCurl{Log: log.DefaultLogger}, + ctx: ctx, + ContentType: "application/json", + InfluxCompatible: true, + UseNativeOr: true, + Timeout: time.Minute, + Curl: &curl.HttpCurl{Log: log.DefaultLogger}, } } diff --git a/pkg/unify-query/tsdb/victoriaMetrics/struct.go b/pkg/unify-query/tsdb/victoriaMetrics/struct.go index 7a1af37b8..2f6f7f666 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/struct.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/struct.go @@ -14,14 +14,6 @@ import ( "strconv" ) -type Params struct { - SQL string `json:"sql"` - BkdataAuthenticationMethod string `json:"bkdata_authentication_method"` - BkAppCode string `json:"bk_app_code"` - PreferStorage string `json:"prefer_storage"` - BkdataDataToken string `json:"bkdata_data_token"` -} - type ParamsQueryRange struct { InfluxCompatible bool `json:"influx_compatible"` UseNativeOr bool `json:"use_native_or"` diff --git a/pkg/unify-query/unify-query.yaml b/pkg/unify-query/unify-query.yaml index e6709b26d..ab14a78c0 100644 --- a/pkg/unify-query/unify-query.yaml +++ b/pkg/unify-query/unify-query.yaml @@ -86,3 +86,11 @@ metadata: druid_query: raw_suffix: "_raw" cmdb_suffix: "_cmdb" +bk_api: + code: + secret: +bk_data: + host: + uri_path: + authentication_method: + token: \ No newline at end of file diff --git a/pkg/utils/router/influxdb/space.go b/pkg/utils/router/influxdb/space.go index accaa2e5f..6a73172e3 100644 --- a/pkg/utils/router/influxdb/space.go +++ b/pkg/utils/router/influxdb/space.go @@ -52,6 +52,7 @@ type ResultTableDetail struct { DataLabel string `json:"data_label"` TagsKey []string `json:"tags_key"` DataId int64 `json:"bk_data_id"` + SourceType string `json:"source_type"` Options struct { // 自定义时间聚合字段 TimeField struct { diff --git a/pkg/utils/router/influxdb/space_gen.go b/pkg/utils/router/influxdb/space_gen.go index 8a1209a73..b0f3b43f0 100644 --- a/pkg/utils/router/influxdb/space_gen.go +++ b/pkg/utils/router/influxdb/space_gen.go @@ -434,6 +434,12 @@ func (z *ResultTableDetail) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "DataId") return } + case "SourceType": + z.SourceType, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SourceType") + return + } case "Options": var zb0004 uint32 zb0004, err = dc.ReadMapHeader() @@ -517,9 +523,9 @@ func (z *ResultTableDetail) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *ResultTableDetail) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 14 + // map header, size 15 // write "StorageId" - err = en.Append(0x8e, 0xa9, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64) + err = en.Append(0x8f, 0xa9, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64) if err != nil { return } @@ -662,6 +668,16 @@ func (z *ResultTableDetail) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "DataId") return } + // write "SourceType" + err = en.Append(0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteString(z.SourceType) + if err != nil { + err = msgp.WrapError(err, "SourceType") + return + } // write "Options" err = en.Append(0xa7, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) if err != nil { @@ -720,9 +736,9 @@ func (z *ResultTableDetail) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *ResultTableDetail) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 14 + // map header, size 15 // string "StorageId" - o = append(o, 0x8e, 0xa9, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64) + o = append(o, 0x8f, 0xa9, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64) o = msgp.AppendInt64(o, z.StorageId) // string "StorageName" o = append(o, 0xab, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65) @@ -766,6 +782,9 @@ func (z *ResultTableDetail) MarshalMsg(b []byte) (o []byte, err error) { // string "DataId" o = append(o, 0xa6, 0x44, 0x61, 0x74, 0x61, 0x49, 0x64) o = msgp.AppendInt64(o, z.DataId) + // string "SourceType" + o = append(o, 0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.SourceType) // string "Options" o = append(o, 0xa7, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) // map header, size 2 @@ -909,6 +928,12 @@ func (z *ResultTableDetail) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "DataId") return } + case "SourceType": + z.SourceType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SourceType") + return + } case "Options": var zb0004 uint32 zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) @@ -1001,7 +1026,7 @@ func (z *ResultTableDetail) Msgsize() (s int) { for za0002 := range z.TagsKey { s += msgp.StringPrefixSize + len(z.TagsKey[za0002]) } - s += 7 + msgp.Int64Size + 8 + 1 + 10 + 1 + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Name) + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Type) + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Unit) + 12 + msgp.BoolSize + s += 7 + msgp.Int64Size + 11 + msgp.StringPrefixSize + len(z.SourceType) + 8 + 1 + 10 + 1 + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Name) + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Type) + 5 + msgp.StringPrefixSize + len(z.Options.TimeField.Unit) + 12 + msgp.BoolSize return } From d776e8c6e50772a0a9c2565e40406c9e3ec22cfc Mon Sep 17 00:00:00 2001 From: Shamcle Ren Date: Thu, 29 Aug 2024 16:06:33 +0800 Subject: [PATCH 004/104] =?UTF-8?q?feat:=20=E8=A7=A3=E5=86=B3=20bksql=20?= =?UTF-8?q?=E9=89=B4=E6=9D=83=E5=8F=82=E6=95=B0=E4=B8=BA=E7=A9=BA=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20--story=3D119438132=20(#511)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/struct.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/struct.go b/pkg/unify-query/tsdb/bksql/struct.go index 63c3c3aee..3147aa196 100644 --- a/pkg/unify-query/tsdb/bksql/struct.go +++ b/pkg/unify-query/tsdb/bksql/struct.go @@ -23,11 +23,8 @@ const ( ) type Params struct { - SQL string `json:"sql"` - BkdataAuthenticationMethod string `json:"bkdata_authentication_method"` - BkAppCode string `json:"bk_app_code"` - PreferStorage string `json:"prefer_storage"` - BkdataDataToken string `json:"bkdata_data_token"` + SQL string `json:"sql"` + PreferStorage string `json:"prefer_storage"` } type Result struct { From 121d75b98cc517185f729767cd5370c475167f41 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Sat, 8 Feb 2025 21:24:06 +0800 Subject: [PATCH 005/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/log/hook.go | 2 +- .../tsdb/elasticsearch/query_string.go | 12 +- .../tsdb/elasticsearch/query_string_test.go | 10 +- .../tsdb/elasticsearch/querystring/expr.go | 174 ++++ .../tsdb/elasticsearch/querystring/lex.go | 320 +++++++ .../tsdb/elasticsearch/querystring/parser.go | 76 ++ .../elasticsearch/querystring/parser_test.go | 265 ++++++ .../elasticsearch/querystring/querystring.y | 270 ++++++ .../querystring/querystring.y.go | 816 ++++++++++++++++++ .../querystring/querystring.y.old | 270 ++++++ .../tsdb/elasticsearch/querystring/y.output | 679 +++++++++++++++ 11 files changed, 2880 insertions(+), 14 deletions(-) create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/expr.go create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/lex.go create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/parser.go create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old create mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/y.output diff --git a/pkg/unify-query/log/hook.go b/pkg/unify-query/log/hook.go index c89a2ae9b..cbcc05f46 100644 --- a/pkg/unify-query/log/hook.go +++ b/pkg/unify-query/log/hook.go @@ -113,6 +113,6 @@ func init() { // InitTestLogger 加载单元测试日志配置 func InitTestLogger() { // 加载配置 - viper.Set(LevelConfigPath, "warn") + viper.Set(LevelConfigPath, "debug") initLogConfig() } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index df9d4abfe..0c6210a48 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -49,22 +49,22 @@ func (s *QueryString) Parser() (elastic.Query, error) { } // 解析失败，或者没有 nested 字段，则使用透传的方式查询 - qs := s.queryString(s.q) + //qs := s.queryString(s.q) ast, err := parser.Parse(s.q) if err != nil { - return qs, nil + return nil, err } conditionQuery, err := s.walk(ast) if err != nil { - return qs, nil + return nil, err } // 如果 nestedFields 不存在则直接使用 queryString 透传 - if len(s.nestedFields) == 0 { - return qs, nil - } + //if len(s.nestedFields) == 0 { + // return qs, nil + //} for nestedKey := range s.nestedFields { conditionQuery = elastic.NewNestedQuery(nestedKey, conditionQuery) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 9b28e3266..5687431ff 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -32,19 +32,15 @@ func TestQsToDsl(t *testing.T) { }{ { q: `log: "ERROR MSG"`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"log: \"ERROR MSG\""}}`, + expected: `{"match_phrase":{"log":{"query":"ERROR MSG"}}}`, }, { q: `quick brown fox`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"quick brown fox"}}`, + expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"quick\""}},{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"brown\""}},{"query_string":{"analyze_wildcard":true,"query":"\"fox\""}}]}}]}}`, }, { q: `word.key: qu?ck`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"word.key: qu?ck"}}`, - }, - { - q: "\"message queue conflict\"", - expected: `{"query_string":{"analyze_wildcard":true,"query":"\"message queue conflict\""}}`, + expected: `{"wildcard":{"word.key":{"value":"qu?ck"}}}`, }, { q: "\"message queue conflict\"", diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/expr.go b/pkg/unify-query/tsdb/elasticsearch/querystring/expr.go new file mode 100644 index 000000000..b9e8ea175 --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/expr.go @@ -0,0 +1,174 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +import ( + "strings" + "time" +) + +// Expr . +type Expr interface { +} + +// AndExpr . +type AndExpr struct { + Left Expr + Right Expr +} + +// NewAndExpr . +func NewAndExpr(left, right Expr) *AndExpr { + return &AndExpr{Left: left, Right: right} +} + +// OrExpr . +type OrExpr struct { + Left Expr + Right Expr +} + +// NewOrExpr . +func NewOrExpr(left, right Expr) *OrExpr { + return &OrExpr{Left: left, Right: right} +} + +// NotExpr . +type NotExpr struct { + Expr Expr +} + +// NewNotExpr . +func NewNotExpr(q Expr) *NotExpr { + return &NotExpr{Expr: q} +} + +// FieldableExpr . +type FieldableExpr interface { + SetField(field string) +} + +// MatchExpr . +type MatchExpr struct { + Field string + Value string +} + +// NewMatchExpr . +func NewMatchExpr(s string) *MatchExpr { + return &MatchExpr{ + Value: s, + } +} + +// SetField . +func (q *MatchExpr) SetField(field string) { + q.Field = field +} + +// RegexpExpr . +type RegexpExpr struct { + Field string + Value string +} + +// NewRegexpExpr . +func NewRegexpExpr(s string) *RegexpExpr { + return &RegexpExpr{ + Value: s, + } +} + +// SetField . +func (q *RegexpExpr) SetField(field string) { + q.Field = field +} + +// WildcardExpr . +type WildcardExpr struct { + Field string + Value string +} + +// NewWildcardExpr . +func NewWildcardExpr(s string) *WildcardExpr { + return &WildcardExpr{ + Value: s, + } +} + +// SetField . +func (q *WildcardExpr) SetField(field string) { + q.Field = field +} + +// NumberRangeExpr . +type NumberRangeExpr struct { + Field string + Start *string + End *string + IncludeStart bool + IncludeEnd bool +} + +// NewNumberRangeExpr . +func NewNumberRangeExpr(start, end *string, includeStart, includeEnd bool) *NumberRangeExpr { + return &NumberRangeExpr{ + Start: start, + End: end, + IncludeStart: includeStart, + IncludeEnd: includeEnd, + } +} + +// SetField . +func (q *NumberRangeExpr) SetField(field string) { + q.Field = field +} + +// TimeRangeExpr . +type TimeRangeExpr struct { + Field string + Start *string + End *string + IncludeStart bool + IncludeEnd bool +} + +// NewTimeRangeExpr . +func NewTimeRangeExpr(start, end *string, includeStart, includeEnd bool) *TimeRangeExpr { + return &TimeRangeExpr{ + Start: start, + End: end, + IncludeStart: includeStart, + IncludeEnd: includeEnd, + } +} + +// SetField . +func (q *TimeRangeExpr) SetField(field string) { + q.Field = field +} + +func queryTimeFromString(t string) (time.Time, error) { + return time.Parse(time.RFC3339, t) +} + +func newStringExpr(str string) FieldableExpr { + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + return NewRegexpExpr(str[1 : len(str)-1]) + } + + if strings.ContainsAny(str, "*?") { + return NewWildcardExpr(str) + } + + return NewMatchExpr(str) +} diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/lex.go b/pkg/unify-query/tsdb/elasticsearch/querystring/lex.go new file mode 100644 index 000000000..ada6eef07 --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/lex.go @@ -0,0 +1,320 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package querystring + +import ( + "bufio" + "bytes" + "context" + "io" + "strings" + "unicode" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" +) + +const reservedChars = "+-=&|>', '<', '=', '(', ')', '[', ']', '{', '}', '*': + l.buf += string(next) + return singleCharOpState, true + } + + switch { + case !l.inEscape && next == '\\': + l.inEscape = true + return startState, true + case unicode.IsDigit(next): + l.buf += string(next) + return inNumOrStrState, true + case !unicode.IsSpace(next): + l.buf += string(next) + return inStrState, true + } + + // doesn't look like anything, just eat it and stay here + l.reset() + return startState, true +} + +func inPhraseState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // unterminated phrase eats the phrase + if eof { + l.Error("unterminated quote") + return nil, false + } + + // only a non-escaped " ends the phrase + if !l.inEscape && (next == '"' || next == '/') { + // end phrase + switch next { + case '"': + l.nextTokenType = tPHRASE + case '/': + l.nextTokenType = tSLASH + } + l.nextToken = &yySymType{ + s: l.buf, + } + log.Debugf(context.TODO(), "PHRASE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inPhraseState, true +} + +func singleCharOpState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + l.nextToken = &yySymType{} + + var msg bytes.Buffer + msg.WriteString(l.buf) + msg.WriteString(" => ") + switch l.buf { + case "+": + l.nextTokenType = tPLUS + msg.WriteString("tPLUS") + case "-": + l.nextTokenType = tMINUS + msg.WriteString("tMINUS") + case ":": + l.nextTokenType = tCOLON + msg.WriteString("tCOLON") + case ">": + l.nextTokenType = tGREATER + msg.WriteString("tGREATER") + case "<": + l.nextTokenType = tLESS + msg.WriteString("tLESS") + case "=": + l.nextTokenType = tEQUAL + msg.WriteString("tEQUAL") + case "(": + l.nextTokenType = tLEFTBRACKET + msg.WriteString("tLEFTBRACKET") + case ")": + l.nextTokenType = tRIGHTBRACKET + msg.WriteString("tRIGHTBRACKET") + case "[": + l.nextTokenType = tLEFTRANGE + msg.WriteString("tLEFTRANGE") + case "]": + l.nextTokenType = tRIGHTRANGE + msg.WriteString("tRIGHTRANGE") + case "{": + l.nextTokenType = tLEFTBRACES + msg.WriteString("tLEFTBRACES") + case "}": + l.nextTokenType = tRIGHTBRACES + msg.WriteString("tRIGHTBRACES") + case "*": + l.nextTokenType = tSTAR + msg.WriteString("tSTAR") + } + log.Debugf(context.TODO(), msg.String()) + + l.reset() + return startState, false +} + +func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // only a non-escaped space ends the tilde (or eof) + if eof || (!l.inEscape && next == ' ' || next == ':' || next == ')' || next == ']' || next == '}') { + // end number + consumed := true + if !eof && (next == ':' || next == ')' || next == ']' || next == '}') { + consumed = false + } + + l.nextTokenType = tNUMBER + l.nextToken = &yySymType{ + s: l.buf, + } + log.Debugf(context.TODO(), "NUMBER - '%s'", l.nextToken.s) + l.reset() + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + return inNumOrStrState, true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + // go directly to string, no successfully or unsuccessfully + // escaped string results in a valid number + return inStrState, true + } + + // see where to go + if !l.seenDot && next == '.' { + // stay in this state + l.seenDot = true + l.buf += string(next) + return inNumOrStrState, true + } else if unicode.IsDigit(next) { + l.buf += string(next) + return inNumOrStrState, true + } + + // doesn't look like an number, transition + l.buf += string(next) + return inStrState, true +} + +func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // end on non-escped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == ')' || next == ']')) { + // end string + consumed := true + if !eof && (next == ':' || next == ')' || next == ']') { + consumed = false + } + + switch strings.ToLower(l.buf) { + case "and": + l.nextTokenType = tAND + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "or": + l.nextTokenType = tOR + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "not": + l.nextTokenType = tNOT + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + case "to": + l.nextTokenType = tTO + l.nextToken = &yySymType{} + l.reset() + return startState, consumed + } + + l.nextTokenType = tSTRING + l.nextToken = &yySymType{ + s: l.buf, + } + log.Debugf(context.TODO(), "STRING - '%s'", l.nextToken.s) + l.reset() + + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inStrState, true +} diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/parser.go b/pkg/unify-query/tsdb/elasticsearch/querystring/parser.go new file mode 100644 index 000000000..0f0ab5d95 --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/parser.go @@ -0,0 +1,76 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +//go:generate goyacc -o querystring.y.go querystring.y + +import ( + "fmt" + "strings" +) + +var debugLexer bool + +// Parse querystring and return Expr +func Parse(query string) (Expr, error) { + if query == "" { + return nil, nil + } + + lex := newLexerWrapper(newExprStringLex(strings.NewReader(query))) + doParse(lex) + + if len(lex.errs) > 0 { + return nil, fmt.Errorf(strings.Join(lex.errs, "\n")) + } + return lex.expr, nil +} + +func doParse(lex *lexerWrapper) { + defer func() { + r := recover() + if r != nil { + lex.errs = append(lex.errs, fmt.Sprintf("parse error: %v", r)) + } + }() + + yyParse(lex) +} + +const ( + queryShould = iota + queryMust + queryMustNot +) + +type lexerWrapper struct { + lex yyLexer + errs []string + expr Expr +} + +func newLexerWrapper(lex yyLexer) *lexerWrapper { + return &lexerWrapper{ + lex: lex, + expr: nil, + } +} + +func (l *lexerWrapper) Lex(lval *yySymType) int { + return l.lex.Lex(lval) +} + +func (l *lexerWrapper) Error(s string) { + l.errs = append(l.errs, s) +} + +func init() { + yyErrorVerbose = true +} diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go b/pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go new file mode 100644 index 000000000..659fb2dcb --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go @@ -0,0 +1,265 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package querystring + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" +) + +/* +精确匹配(支持AND、OR)： +author:"John Smith" AND age:20 +字段名匹配(*代表通配符)： +status:active +title:(quick brown) +字段名模糊匹配： +vers\*on:(quick brown) +通配符匹配： +qu?ck bro* +正则匹配： +name:/joh?n(ath[oa]n)/ +范围匹配： +count:[1 TO 5] +count:[1 TO 5} +count:[10 TO *] +*/ +func TestParser(t *testing.T) { + testCases := map[string]struct { + q string + e Expr + }{ + "正常查询": { + q: `test`, + e: &MatchExpr{ + Value: `test`, + }, + }, + "负数查询": { + q: `-test`, + e: &NotExpr{ + Expr: &MatchExpr{ + Value: `test`, + }, + }, + }, + "负数查询多条件": { + q: `-test AND good`, + e: &AndExpr{ + Left: &NotExpr{ + Expr: &MatchExpr{ + Value: `test`, + }, + }, + Right: &MatchExpr{ + Value: `good`, + }, + }, + }, + "通配符匹配": { + q: `qu?ck bro*`, + e: &AndExpr{ + Left: &WildcardExpr{ + Value: "qu?ck", + }, + Right: &WildcardExpr{ + Value: "bro*", + }, + }, + }, + "正则匹配": { + q: `name:/joh?n(ath[oa]n)/`, + e: &RegexpExpr{ + Field: "name", + Value: "joh?n(ath[oa]n)", + }, + }, + "范围匹配，左闭右开": { + q: `count:[1 TO 5}`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("1"), + End: pointer("5"), + IncludeStart: true, + IncludeEnd: false, + }, + }, + "范围匹配": { + q: `count:[1 TO 5]`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("1"), + End: pointer("5"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + "范围匹配（无上限）": { + q: `count:[10 TO *]`, + e: &NumberRangeExpr{ + Field: "count", + Start: pointer("10"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + "字段匹配": { + q: `status:active`, + e: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + "字段匹配 + 括号": { + q: `status:(active)`, + e: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + "多条件组合，括号调整优先级": { + q: `author:"John Smith" AND (age:20 OR status:active)`, + e: &AndExpr{ + Left: &MatchExpr{ + Field: "author", + Value: "John Smith", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Field: "age", + Value: "20", + }, + Right: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + }, + }, + "多条件组合，and 和 or 的优先级": { + q: `(author:"John Smith" AND age:20) OR status:active`, + e: &OrExpr{ + Left: &AndExpr{ + Left: &MatchExpr{ + Field: "author", + Value: "John Smith", + }, + Right: &MatchExpr{ + Field: "age", + Value: "20", + }, + }, + Right: &MatchExpr{ + Field: "status", + Value: "active", + }, + }, + }, + "嵌套逻辑表达式": { + q: `a:1 AND (b:2 OR c:3)`, + e: &AndExpr{ + Left: &MatchExpr{ + Field: "a", + Value: "1", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Field: "b", + Value: "2", + }, + Right: &MatchExpr{ + Field: "c", + Value: "3", + }, + }, + }, + }, + "模糊匹配": { + q: `quick brown fox`, + e: &AndExpr{ + Left: &MatchExpr{ + Value: "quick", + }, + Right: &AndExpr{ + Left: &MatchExpr{ + Value: "brown", + }, + Right: &MatchExpr{ + Value: "fox", + }, + }, + }, + }, + "单个条件精确匹配": { + q: `log: "ERROR MSG"`, + e: &MatchExpr{ + Field: "log", + Value: "ERROR MSG", + }, + }, + "match and time range": { + q: "message: test\\ value AND datetime: [\"2020-01-01T00:00:00\" TO \"2020-12-31T00:00:00\"]", + e: &AndExpr{ + Left: &MatchExpr{ + Field: "message", + Value: "test value", + }, + Right: &TimeRangeExpr{ + Field: "datetime", + Start: pointer("2020-01-01T00:00:00"), + End: pointer("2020-12-31T00:00:00"), + IncludeStart: true, + IncludeEnd: true, + }, + }, + }, + "mixed or / and": { + q: "a: 1 OR (b: 2 and c: 4)", + e: &OrExpr{ + Left: &MatchExpr{ + Field: "a", + Value: "1", + }, + Right: &AndExpr{ + Left: &MatchExpr{ + Field: "b", + Value: "2", + }, + Right: &MatchExpr{ + Field: "c", + Value: "4", + }, + }, + }, + }, + } + + metadata.InitMetadata() + mock.Init() + + for name, c := range testCases { + t.Run(name, func(t *testing.T) { + expr, err := Parse(c.q) + if err != nil { + t.Errorf("parse return error, %s", err) + return + } + assert.Equal(t, c.e, expr) + }) + } +} + +func pointer(s string) *string { + return &s +} diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y new file mode 100644 index 000000000..26a9aadfd --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y @@ -0,0 +1,270 @@ +%{ +package querystring +%} + +%union { + s string + n int + e Expr +} + +%token tSTRING tPHRASE tNUMBER tSLASH tSTAR +%token tOR tAND tNOT tTO tPLUS tMINUS tCOLON +%token tLEFTBRACKET tRIGHTBRACKET tLEFTRANGE tRIGHTRANGE tLEFTBRACES tRIGHTBRACES +%token tGREATER tLESS tEQUAL + +%type tSTRING +%type tPHRASE +%type tNUMBER +%type tSTAR +%type tSLASH +%type posOrNegNumber +%type searchBase searchLogicParts searchPart searchLogicPart searchLogicSimplePart +%type searchPrefix + +%left tOR +%left tAND +%nonassoc tLEFTBRACKET tRIGHTBRACKET + +%% + +input: +searchLogicParts { + yylex.(*lexerWrapper).expr = $1 +}; + +searchLogicParts: +searchLogicPart searchLogicParts { + $$ = NewAndExpr($1, $2) +} +| +searchLogicPart { + $$ = $1 +} + +searchLogicPart: +searchLogicSimplePart { + $$ = $1 +} +| +searchLogicSimplePart tOR searchLogicPart { + $$ = NewOrExpr($1, $3) +} +| +searchLogicSimplePart tAND searchLogicPart { + $$ = NewAndExpr($1, $3) +}; + +searchLogicSimplePart: +searchPart { + $$ = $1 +} +| +tLEFTBRACKET searchLogicPart tRIGHTBRACKET { + $$ = $2 +} +| +tNOT searchLogicSimplePart { + $$ = NewNotExpr($2) +}; + +searchPart: +searchPrefix searchBase { + switch($1) { + case queryMustNot: + $$ = NewNotExpr($2) + default: + $$ = $2 + } +} +| +searchBase { + $$ = $1 +}; + +searchPrefix: +tPLUS { + $$ = queryMust +} +| +tMINUS { + $$ = queryMustNot +}; + +searchBase: +tSTRING { + $$ = newStringExpr($1) +} +| +tNUMBER { + $$ = NewMatchExpr($1) +} +| +tPHRASE { + phrase := $1 + q := NewMatchExpr(phrase) + $$ = q +} +| +tSLASH{ + phrase := $1 + q := NewRegexpExpr(phrase) + $$ = q +} +| +tSTRING tCOLON tSTRING { + q := newStringExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET { + q := newStringExpr($4) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON posOrNegNumber { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tPHRASE { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tSLASH { + q := NewRegexpExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(&val, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(&val, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(nil, &val, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(nil, &val, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(&phrase, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(&phrase, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(nil, &phrase, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(nil, &phrase, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE { + max := $6 + q := NewNumberRangeExpr(nil, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE { + min := $4 + q := NewNumberRangeExpr(&min, nil, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +}; + +posOrNegNumber: +tNUMBER { + $$ = $1 +} +| +tMINUS tNUMBER { + $$ = "-" + $2 +}; diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go new file mode 100644 index 000000000..27d629a3b --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go @@ -0,0 +1,816 @@ +// Code generated by goyacc -o querystring.y.go querystring.y. DO NOT EDIT. + +//line querystring.y:2 +package querystring + +import __yyfmt__ "fmt" + +//line querystring.y:2 + +//line querystring.y:5 +type yySymType struct { + yys int + s string + n int + e Expr +} + +const tSTRING = 57346 +const tPHRASE = 57347 +const tNUMBER = 57348 +const tSLASH = 57349 +const tSTAR = 57350 +const tOR = 57351 +const tAND = 57352 +const tNOT = 57353 +const tTO = 57354 +const tPLUS = 57355 +const tMINUS = 57356 +const tCOLON = 57357 +const tLEFTBRACKET = 57358 +const tRIGHTBRACKET = 57359 +const tLEFTRANGE = 57360 +const tRIGHTRANGE = 57361 +const tLEFTBRACES = 57362 +const tRIGHTBRACES = 57363 +const tGREATER = 57364 +const tLESS = 57365 +const tEQUAL = 57366 + +var yyToknames = [...]string{ + "$end", + "error", + "$unk", + "tSTRING", + "tPHRASE", + "tNUMBER", + "tSLASH", + "tSTAR", + "tOR", + "tAND", + "tNOT", + "tTO", + "tPLUS", + "tMINUS", + "tCOLON", + "tLEFTBRACKET", + "tRIGHTBRACKET", + "tLEFTRANGE", + "tRIGHTRANGE", + "tLEFTBRACES", + "tRIGHTBRACES", + "tGREATER", + "tLESS", + "tEQUAL", +} + +var yyStatenames = [...]string{} + +const yyEofCode = 1 +const yyErrCode = 2 +const yyInitialStackSize = 16 + +//line yacctab:1 +var yyExca = [...]int8{ + -1, 1, + 1, -1, + -2, 0, +} + +const yyPrivate = 57344 + +const yyLast = 93 + +var yyAct = [...]int8{ + 28, 26, 29, 35, 30, 43, 35, 40, 35, 73, + 71, 36, 70, 27, 36, 33, 36, 34, 72, 31, + 32, 22, 67, 69, 42, 68, 39, 66, 50, 3, + 25, 59, 38, 41, 45, 47, 19, 35, 35, 61, + 51, 46, 35, 53, 44, 36, 36, 23, 24, 58, + 36, 12, 14, 13, 15, 57, 60, 62, 7, 64, + 10, 11, 56, 6, 54, 35, 52, 35, 48, 35, + 55, 17, 18, 36, 9, 36, 4, 36, 12, 14, + 13, 15, 49, 21, 20, 65, 63, 37, 2, 1, + 8, 5, 16, +} + +var yyPact = [...]int16{ + 47, -1000, -1000, 47, 62, -1000, 47, 47, 74, -1000, + -1000, -1000, 6, -1000, -1000, -1000, -1000, 47, 47, 13, + -1000, -1000, -3, -1000, -1000, -1000, -1000, 83, -1000, -1000, + -1000, 2, 0, 36, 63, -1000, 76, 11, -1000, 61, + -1000, -1000, 59, -1000, 58, 50, 43, 37, 19, -1000, + -1000, -1000, -1000, -1000, -1000, 32, 31, 81, 32, 80, + 8, 3, 4, -9, -1, -10, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, +} + +var yyPgo = [...]int8{ + 0, 0, 74, 88, 91, 29, 76, 90, 89, +} + +var yyR1 = [...]int8{ + 0, 8, 3, 3, 5, 5, 5, 6, 6, 6, + 4, 4, 7, 7, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, + 1, +} + +var yyR2 = [...]int8{ + 0, 1, 2, 1, 1, 3, 3, 1, 3, 2, + 2, 1, 1, 1, 1, 1, 1, 1, 3, 5, + 3, 3, 3, 4, 5, 4, 5, 4, 5, 4, + 5, 7, 7, 7, 7, 7, 7, 7, 7, 1, + 2, +} + +var yyChk = [...]int16{ + -1000, -8, -3, -5, -6, -4, 16, 11, -7, -2, + 13, 14, 4, 6, 5, 7, -3, 9, 10, -5, + -6, -2, 15, -5, -5, 17, 4, 16, -1, 5, + 7, 22, 23, 18, 20, 6, 14, 4, -1, 24, + 5, -1, 24, 5, 8, -1, 5, -1, 5, 6, + 17, -1, 5, -1, 5, 12, 12, 12, 12, 12, + -1, 8, -1, 5, -1, 5, 19, 19, 21, 19, + 21, 19, 19, 19, +} + +var yyDef = [...]int8{ + 0, -2, 1, 3, 4, 7, 0, 0, 0, 11, + 12, 13, 14, 15, 16, 17, 2, 0, 0, 0, + 9, 10, 0, 5, 6, 8, 18, 0, 20, 21, + 22, 0, 0, 0, 0, 39, 0, 0, 23, 0, + 27, 25, 0, 29, 0, 0, 0, 0, 0, 40, + 19, 24, 28, 26, 30, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 31, 32, 33, 37, + 34, 38, 35, 36, +} + +var yyTok1 = [...]int8{ + 1, +} + +var yyTok2 = [...]int8{ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, +} + +var yyTok3 = [...]int8{ + 0, +} + +var yyErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + yyDebug = 0 + yyErrorVerbose = false +) + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyParser interface { + Parse(yyLexer) int + Lookahead() int +} + +type yyParserImpl struct { + lval yySymType + stack [yyInitialStackSize]yySymType + char int +} + +func (p *yyParserImpl) Lookahead() int { + return p.char +} + +func yyNewParser() yyParser { + return &yyParserImpl{} +} + +const yyFlag = -1000 + +func yyTokname(c int) string { + if c >= 1 && c-1 < len(yyToknames) { + if yyToknames[c-1] != "" { + return yyToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func yyStatname(s int) string { + if s >= 0 && s < len(yyStatenames) { + if yyStatenames[s] != "" { + return yyStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func yyErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !yyErrorVerbose { + return "syntax error" + } + + for _, e := range yyErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + yyTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := int(yyPact[state]) + for tok := TOKSTART; tok-1 < len(yyToknames); tok++ { + if n := base + tok; n >= 0 && n < yyLast && int(yyChk[int(yyAct[n])]) == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if yyDef[state] == -2 { + i := 0 + for yyExca[i] != -1 || int(yyExca[i+1]) != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; yyExca[i] >= 0; i += 2 { + tok := int(yyExca[i]) + if tok < TOKSTART || yyExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if yyExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += yyTokname(tok) + } + return res +} + +func yylex1(lex yyLexer, lval *yySymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = int(yyTok1[0]) + goto out + } + if char < len(yyTok1) { + token = int(yyTok1[char]) + goto out + } + if char >= yyPrivate { + if char < yyPrivate+len(yyTok2) { + token = int(yyTok2[char-yyPrivate]) + goto out + } + } + for i := 0; i < len(yyTok3); i += 2 { + token = int(yyTok3[i+0]) + if token == char { + token = int(yyTok3[i+1]) + goto out + } + } + +out: + if token == 0 { + token = int(yyTok2[1]) /* unknown char */ + } + if yyDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char)) + } + return char, token +} + +func yyParse(yylex yyLexer) int { + return yyNewParser().Parse(yylex) +} + +func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int { + var yyn int + var yyVAL yySymType + var yyDollar []yySymType + _ = yyDollar // silence set and not used + yyS := yyrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yystate := 0 + yyrcvr.char = -1 + yytoken := -1 // yyrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + yystate = -1 + yyrcvr.char = -1 + yytoken = -1 + }() + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + if yyDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate)) + } + + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + yyn = int(yyPact[yystate]) + if yyn <= yyFlag { + goto yydefault /* simple state */ + } + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + yyn += yytoken + if yyn < 0 || yyn >= yyLast { + goto yydefault + } + yyn = int(yyAct[yyn]) + if int(yyChk[yyn]) == yytoken { /* valid shift */ + yyrcvr.char = -1 + yytoken = -1 + yyVAL = yyrcvr.lval + yystate = yyn + if Errflag > 0 { + Errflag-- + } + goto yystack + } + +yydefault: + /* default state action */ + yyn = int(yyDef[yystate]) + if yyn == -2 { + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if yyExca[xi+0] == -1 && int(yyExca[xi+1]) == yystate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + yyn = int(yyExca[xi+0]) + if yyn < 0 || yyn == yytoken { + break + } + } + yyn = int(yyExca[xi+1]) + if yyn < 0 { + goto ret0 + } + } + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + yylex.Error(yyErrorMessage(yystate, yytoken)) + Nerrs++ + if yyDebug >= 1 { + __yyfmt__.Printf("%s", yyStatname(yystate)) + __yyfmt__.Printf(" saw %s\n", yyTokname(yytoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + yyn = int(yyPact[yyS[yyp].yys]) + yyErrCode + if yyn >= 0 && yyn < yyLast { + yystate = int(yyAct[yyn]) /* simulate a shift of "error" */ + if int(yyChk[yystate]) == yyErrCode { + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken)) + } + if yytoken == yyEofCode { + goto ret1 + } + yyrcvr.char = -1 + yytoken = -1 + goto yynewstate /* try again in the same state */ + } + } + + /* reduction by production yyn */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + } + + yynt := yyn + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= int(yyR2[yyn]) + // yyp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + yyn = int(yyR1[yyn]) + yyg := int(yyPgo[yyn]) + yyj := yyg + yyS[yyp].yys + 1 + + if yyj >= yyLast { + yystate = int(yyAct[yyg]) + } else { + yystate = int(yyAct[yyj]) + if int(yyChk[yystate]) != -yyn { + yystate = int(yyAct[yyg]) + } + } + // dummy call; replaced with literal code + switch yynt { + + case 1: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:32 + { + yylex.(*lexerWrapper).expr = yyDollar[1].e + } + case 2: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:37 + { + yyVAL.e = NewAndExpr(yyDollar[1].e, yyDollar[2].e) + } + case 3: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:41 + { + yyVAL.e = yyDollar[1].e + } + case 4: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:46 + { + yyVAL.e = yyDollar[1].e + } + case 5: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:50 + { + yyVAL.e = NewOrExpr(yyDollar[1].e, yyDollar[3].e) + } + case 6: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:54 + { + yyVAL.e = NewAndExpr(yyDollar[1].e, yyDollar[3].e) + } + case 7: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:59 + { + yyVAL.e = yyDollar[1].e + } + case 8: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:63 + { + yyVAL.e = yyDollar[2].e + } + case 9: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:67 + { + yyVAL.e = NewNotExpr(yyDollar[2].e) + } + case 10: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:72 + { + switch yyDollar[1].n { + case queryMustNot: + yyVAL.e = NewNotExpr(yyDollar[2].e) + default: + yyVAL.e = yyDollar[2].e + } + } + case 11: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:81 + { + yyVAL.e = yyDollar[1].e + } + case 12: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:86 + { + yyVAL.n = queryMust + } + case 13: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:90 + { + yyVAL.n = queryMustNot + } + case 14: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:95 + { + yyVAL.e = newStringExpr(yyDollar[1].s) + } + case 15: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:99 + { + yyVAL.e = NewMatchExpr(yyDollar[1].s) + } + case 16: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:103 + { + phrase := yyDollar[1].s + q := NewMatchExpr(phrase) + yyVAL.e = q + } + case 17: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:109 + { + phrase := yyDollar[1].s + q := NewRegexpExpr(phrase) + yyVAL.e = q + } + case 18: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:115 + { + q := newStringExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 19: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:121 + { + q := newStringExpr(yyDollar[4].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 20: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:127 + { + q := NewMatchExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 21: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:133 + { + q := NewMatchExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 22: + yyDollar = yyS[yypt-3 : yypt+1] +//line querystring.y:139 + { + q := NewRegexpExpr(yyDollar[3].s) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 23: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:145 + { + val := yyDollar[4].s + q := NewNumberRangeExpr(&val, nil, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 24: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:152 + { + val := yyDollar[5].s + q := NewNumberRangeExpr(&val, nil, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 25: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:159 + { + val := yyDollar[4].s + q := NewNumberRangeExpr(nil, &val, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 26: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:166 + { + val := yyDollar[5].s + q := NewNumberRangeExpr(nil, &val, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 27: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:173 + { + phrase := yyDollar[4].s + q := NewTimeRangeExpr(&phrase, nil, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 28: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:180 + { + phrase := yyDollar[5].s + q := NewTimeRangeExpr(&phrase, nil, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 29: + yyDollar = yyS[yypt-4 : yypt+1] +//line querystring.y:187 + { + phrase := yyDollar[4].s + q := NewTimeRangeExpr(nil, &phrase, false, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 30: + yyDollar = yyS[yypt-5 : yypt+1] +//line querystring.y:194 + { + phrase := yyDollar[5].s + q := NewTimeRangeExpr(nil, &phrase, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 31: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:201 + { + max := yyDollar[6].s + q := NewNumberRangeExpr(nil, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 32: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:208 + { + min := yyDollar[4].s + q := NewNumberRangeExpr(&min, nil, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 33: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:215 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 34: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:223 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, true, false) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 35: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:231 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 36: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:239 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, false, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 37: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:247 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewNumberRangeExpr(&min, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 38: + yyDollar = yyS[yypt-7 : yypt+1] +//line querystring.y:255 + { + min := yyDollar[4].s + max := yyDollar[6].s + q := NewTimeRangeExpr(&min, &max, true, true) + q.SetField(yyDollar[1].s) + yyVAL.e = q + } + case 39: + yyDollar = yyS[yypt-1 : yypt+1] +//line querystring.y:264 + { + yyVAL.s = yyDollar[1].s + } + case 40: + yyDollar = yyS[yypt-2 : yypt+1] +//line querystring.y:268 + { + yyVAL.s = "-" + yyDollar[2].s + } + } + goto yystack /* stack new state and value */ +} diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old new file mode 100644 index 000000000..c87c05ef8 --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old @@ -0,0 +1,270 @@ +%{ +package querystring +%} + +%union { + s string + n int + e Expr +} + +%token tSTRING tPHRASE tNUMBER tSLASH tSTAR +%token tOR tAND tNOT tTO tPLUS tMINUS tCOLON +%token tLEFTBRACKET tRIGHTBRACKET tLEFTRANGE tRIGHTRANGE tLEFTBRACES tRIGHTBRACES +%token tGREATER tLESS tEQUAL + +%type tSTRING +%type tPHRASE +%type tNUMBER +%type tSTAR +%type tSLASH +%type posOrNegNumber +%type searchBase searchParts searchPart searchLogicPart searchLogicSimplePart +%type searchPrefix + +%left tOR +%left tAND +%nonassoc tLEFTBRACKET tRIGHTBRACKET + +%% + +input: +searchParts { + yylex.(*lexerWrapper).expr = $1 +}; + +searchParts: +searchPart searchParts { + $$ = NewAndExpr($1, $2) +} +| +searchPart { + $$ = $1 +}; + +searchPart: +searchPrefix searchBase { + switch($1) { + case queryMustNot: + $$ = NewNotExpr($2) + default: + $$ = $2 + } +} +| +searchLogicPart { + $$ = $1 +}; + +searchLogicPart: +searchLogicSimplePart { + $$ = $1 +} +| +searchLogicSimplePart tOR searchLogicPart { + $$ = NewOrExpr($1, $3) +} +| +searchLogicSimplePart tAND searchLogicPart { + $$ = NewAndExpr($1, $3) +}; + +searchLogicSimplePart: +searchBase { + $$ = $1 +} +| +tLEFTBRACKET searchLogicPart tRIGHTBRACKET { + $$ = $2 +} +| +tNOT searchLogicSimplePart { + $$ = NewNotExpr($2) +}; + +searchPrefix: +tPLUS { + $$ = queryMust +} +| +tMINUS { + $$ = queryMustNot +}; + +searchBase: +tSTRING { + $$ = newStringExpr($1) +} +| +tNUMBER { + $$ = NewMatchExpr($1) +} +| +tPHRASE { + phrase := $1 + q := NewMatchExpr(phrase) + $$ = q +} +| +tSLASH{ + phrase := $1 + q := NewRegexpExpr(phrase) + $$ = q +} +| +tSTRING tCOLON tSTRING { + q := newStringExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET { + q := newStringExpr($4) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON posOrNegNumber { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tPHRASE { + q := NewMatchExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tSLASH { + q := NewRegexpExpr($3) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(&val, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(&val, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS posOrNegNumber { + val := $4 + q := NewNumberRangeExpr(nil, &val, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL posOrNegNumber { + val := $5 + q := NewNumberRangeExpr(nil, &val, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(&phrase, nil, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(&phrase, nil, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tPHRASE { + phrase := $4 + q := NewTimeRangeExpr(nil, &phrase, false, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL tPHRASE { + phrase := $5 + q := NewTimeRangeExpr(nil, &phrase, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE { + max := $6 + q := NewNumberRangeExpr(nil, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE { + min := $4 + q := NewNumberRangeExpr(&min, nil, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, false) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, false, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { + min := $4 + max := $6 + q := NewNumberRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +} +| +tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE { + min := $4 + max := $6 + q := NewTimeRangeExpr(&min, &max, true, true) + q.SetField($1) + $$ = q +}; + +posOrNegNumber: +tNUMBER { + $$ = $1 +} +| +tMINUS tNUMBER { + $$ = "-" + $2 +}; diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/y.output b/pkg/unify-query/tsdb/elasticsearch/querystring/y.output new file mode 100644 index 000000000..7c82715fc --- /dev/null +++ b/pkg/unify-query/tsdb/elasticsearch/querystring/y.output @@ -0,0 +1,679 @@ + +state 0 + $accept: .input $end + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchLogicParts goto 2 + searchPart goto 5 + searchLogicPart goto 3 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + input goto 1 + +state 1 + $accept: input.$end + + $end accept + . error + + +state 2 + input: searchLogicParts. (1) + + . reduce 1 (src line 31) + + +state 3 + searchLogicParts: searchLogicPart.searchLogicParts + searchLogicParts: searchLogicPart. (3) + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . reduce 3 (src line 40) + + searchBase goto 9 + searchLogicParts goto 16 + searchPart goto 5 + searchLogicPart goto 3 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 4 + searchLogicPart: searchLogicSimplePart. (4) + searchLogicPart: searchLogicSimplePart.tOR searchLogicPart + searchLogicPart: searchLogicSimplePart.tAND searchLogicPart + + tOR shift 17 + tAND shift 18 + . reduce 4 (src line 45) + + +state 5 + searchLogicSimplePart: searchPart. (7) + + . reduce 7 (src line 58) + + +state 6 + searchLogicSimplePart: tLEFTBRACKET.searchLogicPart tRIGHTBRACKET + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 19 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 7 + searchLogicSimplePart: tNOT.searchLogicSimplePart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicSimplePart goto 20 + searchPrefix goto 8 + +state 8 + searchPart: searchPrefix.searchBase + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + . error + + searchBase goto 21 + +state 9 + searchPart: searchBase. (11) + + . reduce 11 (src line 80) + + +state 10 + searchPrefix: tPLUS. (12) + + . reduce 12 (src line 85) + + +state 11 + searchPrefix: tMINUS. (13) + + . reduce 13 (src line 89) + + +state 12 + searchBase: tSTRING. (14) + searchBase: tSTRING.tCOLON tSTRING + searchBase: tSTRING.tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET + searchBase: tSTRING.tCOLON posOrNegNumber + searchBase: tSTRING.tCOLON tPHRASE + searchBase: tSTRING.tCOLON tSLASH + searchBase: tSTRING.tCOLON tGREATER posOrNegNumber + searchBase: tSTRING.tCOLON tGREATER tEQUAL posOrNegNumber + searchBase: tSTRING.tCOLON tLESS posOrNegNumber + searchBase: tSTRING.tCOLON tLESS tEQUAL posOrNegNumber + searchBase: tSTRING.tCOLON tGREATER tPHRASE + searchBase: tSTRING.tCOLON tGREATER tEQUAL tPHRASE + searchBase: tSTRING.tCOLON tLESS tPHRASE + searchBase: tSTRING.tCOLON tLESS tEQUAL tPHRASE + searchBase: tSTRING.tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING.tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING.tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING.tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE + + tCOLON shift 22 + . reduce 14 (src line 94) + + +state 13 + searchBase: tNUMBER. (15) + + . reduce 15 (src line 98) + + +state 14 + searchBase: tPHRASE. (16) + + . reduce 16 (src line 102) + + +state 15 + searchBase: tSLASH. (17) + + . reduce 17 (src line 108) + + +state 16 + searchLogicParts: searchLogicPart searchLogicParts. (2) + + . reduce 2 (src line 36) + + +state 17 + searchLogicPart: searchLogicSimplePart tOR.searchLogicPart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 23 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 18 + searchLogicPart: searchLogicSimplePart tAND.searchLogicPart + + tSTRING shift 12 + tPHRASE shift 14 + tNUMBER shift 13 + tSLASH shift 15 + tNOT shift 7 + tPLUS shift 10 + tMINUS shift 11 + tLEFTBRACKET shift 6 + . error + + searchBase goto 9 + searchPart goto 5 + searchLogicPart goto 24 + searchLogicSimplePart goto 4 + searchPrefix goto 8 + +state 19 + searchLogicSimplePart: tLEFTBRACKET searchLogicPart.tRIGHTBRACKET + + tRIGHTBRACKET shift 25 + . error + + +state 20 + searchLogicSimplePart: tNOT searchLogicSimplePart. (9) + + . reduce 9 (src line 66) + + +state 21 + searchPart: searchPrefix searchBase. (10) + + . reduce 10 (src line 71) + + +state 22 + searchBase: tSTRING tCOLON.tSTRING + searchBase: tSTRING tCOLON.tLEFTBRACKET tSTRING tRIGHTBRACKET + searchBase: tSTRING tCOLON.posOrNegNumber + searchBase: tSTRING tCOLON.tPHRASE + searchBase: tSTRING tCOLON.tSLASH + searchBase: tSTRING tCOLON.tGREATER posOrNegNumber + searchBase: tSTRING tCOLON.tGREATER tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON.tLESS posOrNegNumber + searchBase: tSTRING tCOLON.tLESS tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON.tGREATER tPHRASE + searchBase: tSTRING tCOLON.tGREATER tEQUAL tPHRASE + searchBase: tSTRING tCOLON.tLESS tPHRASE + searchBase: tSTRING tCOLON.tLESS tEQUAL tPHRASE + searchBase: tSTRING tCOLON.tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON.tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON.tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON.tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE + + tSTRING shift 26 + tPHRASE shift 29 + tNUMBER shift 35 + tSLASH shift 30 + tMINUS shift 36 + tLEFTBRACKET shift 27 + tLEFTRANGE shift 33 + tLEFTBRACES shift 34 + tGREATER shift 31 + tLESS shift 32 + . error + + posOrNegNumber goto 28 + +state 23 + searchLogicPart: searchLogicSimplePart tOR searchLogicPart. (5) + + . reduce 5 (src line 49) + + +state 24 + searchLogicPart: searchLogicSimplePart tAND searchLogicPart. (6) + + . reduce 6 (src line 53) + + +state 25 + searchLogicSimplePart: tLEFTBRACKET searchLogicPart tRIGHTBRACKET. (8) + + . reduce 8 (src line 62) + + +state 26 + searchBase: tSTRING tCOLON tSTRING. (18) + + . reduce 18 (src line 114) + + +state 27 + searchBase: tSTRING tCOLON tLEFTBRACKET.tSTRING tRIGHTBRACKET + + tSTRING shift 37 + . error + + +state 28 + searchBase: tSTRING tCOLON posOrNegNumber. (20) + + . reduce 20 (src line 126) + + +state 29 + searchBase: tSTRING tCOLON tPHRASE. (21) + + . reduce 21 (src line 132) + + +state 30 + searchBase: tSTRING tCOLON tSLASH. (22) + + . reduce 22 (src line 138) + + +state 31 + searchBase: tSTRING tCOLON tGREATER.posOrNegNumber + searchBase: tSTRING tCOLON tGREATER.tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON tGREATER.tPHRASE + searchBase: tSTRING tCOLON tGREATER.tEQUAL tPHRASE + + tPHRASE shift 40 + tNUMBER shift 35 + tMINUS shift 36 + tEQUAL shift 39 + . error + + posOrNegNumber goto 38 + +state 32 + searchBase: tSTRING tCOLON tLESS.posOrNegNumber + searchBase: tSTRING tCOLON tLESS.tEQUAL posOrNegNumber + searchBase: tSTRING tCOLON tLESS.tPHRASE + searchBase: tSTRING tCOLON tLESS.tEQUAL tPHRASE + + tPHRASE shift 43 + tNUMBER shift 35 + tMINUS shift 36 + tEQUAL shift 42 + . error + + posOrNegNumber goto 41 + +state 33 + searchBase: tSTRING tCOLON tLEFTRANGE.tSTAR tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE.tPHRASE tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE.posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE.tPHRASE tTO tPHRASE tRIGHTRANGE + + tPHRASE shift 46 + tNUMBER shift 35 + tSTAR shift 44 + tMINUS shift 36 + . error + + posOrNegNumber goto 45 + +state 34 + searchBase: tSTRING tCOLON tLEFTBRACES.posOrNegNumber tTO posOrNegNumber tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTBRACES.tPHRASE tTO tPHRASE tRIGHTRANGE + + tPHRASE shift 48 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 47 + +state 35 + posOrNegNumber: tNUMBER. (39) + + . reduce 39 (src line 263) + + +state 36 + posOrNegNumber: tMINUS.tNUMBER + + tNUMBER shift 49 + . error + + +state 37 + searchBase: tSTRING tCOLON tLEFTBRACKET tSTRING.tRIGHTBRACKET + + tRIGHTBRACKET shift 50 + . error + + +state 38 + searchBase: tSTRING tCOLON tGREATER posOrNegNumber. (23) + + . reduce 23 (src line 144) + + +state 39 + searchBase: tSTRING tCOLON tGREATER tEQUAL.posOrNegNumber + searchBase: tSTRING tCOLON tGREATER tEQUAL.tPHRASE + + tPHRASE shift 52 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 51 + +state 40 + searchBase: tSTRING tCOLON tGREATER tPHRASE. (27) + + . reduce 27 (src line 172) + + +state 41 + searchBase: tSTRING tCOLON tLESS posOrNegNumber. (25) + + . reduce 25 (src line 158) + + +state 42 + searchBase: tSTRING tCOLON tLESS tEQUAL.posOrNegNumber + searchBase: tSTRING tCOLON tLESS tEQUAL.tPHRASE + + tPHRASE shift 54 + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 53 + +state 43 + searchBase: tSTRING tCOLON tLESS tPHRASE. (29) + + . reduce 29 (src line 186) + + +state 44 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 55 + . error + + +state 45 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 56 + . error + + +state 46 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE.tTO tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE.tTO tPHRASE tRIGHTRANGE + + tTO shift 57 + . error + + +state 47 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber.tTO posOrNegNumber tRIGHTRANGE + + tTO shift 58 + . error + + +state 48 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE.tTO tPHRASE tRIGHTRANGE + + tTO shift 59 + . error + + +state 49 + posOrNegNumber: tMINUS tNUMBER. (40) + + . reduce 40 (src line 267) + + +state 50 + searchBase: tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET. (19) + + . reduce 19 (src line 120) + + +state 51 + searchBase: tSTRING tCOLON tGREATER tEQUAL posOrNegNumber. (24) + + . reduce 24 (src line 151) + + +state 52 + searchBase: tSTRING tCOLON tGREATER tEQUAL tPHRASE. (28) + + . reduce 28 (src line 179) + + +state 53 + searchBase: tSTRING tCOLON tLESS tEQUAL posOrNegNumber. (26) + + . reduce 26 (src line 165) + + +state 54 + searchBase: tSTRING tCOLON tLESS tEQUAL tPHRASE. (30) + + . reduce 30 (src line 193) + + +state 55 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 60 + +state 56 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.tSTAR tRIGHTRANGE + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.posOrNegNumber tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tSTAR shift 61 + tMINUS shift 36 + . error + + posOrNegNumber goto 62 + +state 57 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO.tPHRASE tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO.tPHRASE tRIGHTRANGE + + tPHRASE shift 63 + . error + + +state 58 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO.posOrNegNumber tRIGHTRANGE + + tNUMBER shift 35 + tMINUS shift 36 + . error + + posOrNegNumber goto 64 + +state 59 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO.tPHRASE tRIGHTRANGE + + tPHRASE shift 65 + . error + + +state 60 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 66 + . error + + +state 61 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR.tRIGHTRANGE + + tRIGHTRANGE shift 67 + . error + + +state 62 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber.tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 69 + tRIGHTBRACES shift 68 + . error + + +state 63 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE.tRIGHTBRACES + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE.tRIGHTRANGE + + tRIGHTRANGE shift 71 + tRIGHTBRACES shift 70 + . error + + +state 64 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber.tRIGHTRANGE + + tRIGHTRANGE shift 72 + . error + + +state 65 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE.tRIGHTRANGE + + tRIGHTRANGE shift 73 + . error + + +state 66 + searchBase: tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE. (31) + + . reduce 31 (src line 200) + + +state 67 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE. (32) + + . reduce 32 (src line 207) + + +state 68 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES. (33) + + . reduce 33 (src line 214) + + +state 69 + searchBase: tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE. (37) + + . reduce 37 (src line 246) + + +state 70 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES. (34) + + . reduce 34 (src line 222) + + +state 71 + searchBase: tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE. (38) + + . reduce 38 (src line 254) + + +state 72 + searchBase: tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE. (35) + + . reduce 35 (src line 230) + + +state 73 + searchBase: tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE. (36) + + . reduce 36 (src line 238) + + +24 terminals, 9 nonterminals +41 grammar rules, 74/16000 states +0 shift/reduce, 0 reduce/reduce conflicts reported +58 working sets used +memory: parser 42/240000 +12 extra closures +112 shift entries, 1 exceptions +23 goto entries +20 entries saved by goto default +Optimizer space used: output 93/240000 +93 table entries, 0 zero +maximum spread: 24, maximum offset: 58 From df195b620f3ddadb8a9b52f241667927697a0f8a Mon Sep 17 00:00:00 2001 From: shamcleren Date: Sat, 8 Feb 2025 21:30:12 +0800 Subject: [PATCH 006/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../querystring/querystring.y.old | 270 ------------------ 1 file changed, 270 deletions(-) delete mode 100644 pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old b/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old deleted file mode 100644 index c87c05ef8..000000000 --- a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.old +++ /dev/null @@ -1,270 +0,0 @@ -%{ -package querystring -%} - -%union { - s string - n int - e Expr -} - -%token tSTRING tPHRASE tNUMBER tSLASH tSTAR -%token tOR tAND tNOT tTO tPLUS tMINUS tCOLON -%token tLEFTBRACKET tRIGHTBRACKET tLEFTRANGE tRIGHTRANGE tLEFTBRACES tRIGHTBRACES -%token tGREATER tLESS tEQUAL - -%type tSTRING -%type tPHRASE -%type tNUMBER -%type tSTAR -%type tSLASH -%type posOrNegNumber -%type searchBase searchParts searchPart searchLogicPart searchLogicSimplePart -%type searchPrefix - -%left tOR -%left tAND -%nonassoc tLEFTBRACKET tRIGHTBRACKET - -%% - -input: -searchParts { - yylex.(*lexerWrapper).expr = $1 -}; - -searchParts: -searchPart searchParts { - $$ = NewAndExpr($1, $2) -} -| -searchPart { - $$ = $1 -}; - -searchPart: -searchPrefix searchBase { - switch($1) { - case queryMustNot: - $$ = NewNotExpr($2) - default: - $$ = $2 - } -} -| -searchLogicPart { - $$ = $1 -}; - -searchLogicPart: -searchLogicSimplePart { - $$ = $1 -} -| -searchLogicSimplePart tOR searchLogicPart { - $$ = NewOrExpr($1, $3) -} -| -searchLogicSimplePart tAND searchLogicPart { - $$ = NewAndExpr($1, $3) -}; - -searchLogicSimplePart: -searchBase { - $$ = $1 -} -| -tLEFTBRACKET searchLogicPart tRIGHTBRACKET { - $$ = $2 -} -| -tNOT searchLogicSimplePart { - $$ = NewNotExpr($2) -}; - -searchPrefix: -tPLUS { - $$ = queryMust -} -| -tMINUS { - $$ = queryMustNot -}; - -searchBase: -tSTRING { - $$ = newStringExpr($1) -} -| -tNUMBER { - $$ = NewMatchExpr($1) -} -| -tPHRASE { - phrase := $1 - q := NewMatchExpr(phrase) - $$ = q -} -| -tSLASH{ - phrase := $1 - q := NewRegexpExpr(phrase) - $$ = q -} -| -tSTRING tCOLON tSTRING { - q := newStringExpr($3) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTBRACKET tSTRING tRIGHTBRACKET { - q := newStringExpr($4) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON posOrNegNumber { - q := NewMatchExpr($3) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tPHRASE { - q := NewMatchExpr($3) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tSLASH { - q := NewRegexpExpr($3) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tGREATER posOrNegNumber { - val := $4 - q := NewNumberRangeExpr(&val, nil, false, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { - val := $5 - q := NewNumberRangeExpr(&val, nil, true, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLESS posOrNegNumber { - val := $4 - q := NewNumberRangeExpr(nil, &val, false, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLESS tEQUAL posOrNegNumber { - val := $5 - q := NewNumberRangeExpr(nil, &val, false, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tGREATER tPHRASE { - phrase := $4 - q := NewTimeRangeExpr(&phrase, nil, false, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tGREATER tEQUAL tPHRASE { - phrase := $5 - q := NewTimeRangeExpr(&phrase, nil, true, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLESS tPHRASE { - phrase := $4 - q := NewTimeRangeExpr(nil, &phrase, false, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLESS tEQUAL tPHRASE { - phrase := $5 - q := NewTimeRangeExpr(nil, &phrase, false, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE tSTAR tTO posOrNegNumber tRIGHTRANGE { - max := $6 - q := NewNumberRangeExpr(nil, &max, true, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO tSTAR tRIGHTRANGE { - min := $4 - q := NewNumberRangeExpr(&min, nil, true, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTBRACES { - min := $4 - max := $6 - q := NewNumberRangeExpr(&min, &max, true, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTBRACES { - min := $4 - max := $6 - q := NewTimeRangeExpr(&min, &max, true, false) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTBRACES posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { - min := $4 - max := $6 - q := NewNumberRangeExpr(&min, &max, false, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTBRACES tPHRASE tTO tPHRASE tRIGHTRANGE { - min := $4 - max := $6 - q := NewTimeRangeExpr(&min, &max, false, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE posOrNegNumber tTO posOrNegNumber tRIGHTRANGE { - min := $4 - max := $6 - q := NewNumberRangeExpr(&min, &max, true, true) - q.SetField($1) - $$ = q -} -| -tSTRING tCOLON tLEFTRANGE tPHRASE tTO tPHRASE tRIGHTRANGE { - min := $4 - max := $6 - q := NewTimeRangeExpr(&min, &max, true, true) - q.SetField($1) - $$ = q -}; - -posOrNegNumber: -tNUMBER { - $$ = $1 -} -| -tMINUS tNUMBER { - $$ = "-" + $2 -}; From 34f457aaeb72d814b41d9d659d0ddf211e73b947 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Sat, 8 Feb 2025 21:35:25 +0800 Subject: [PATCH 007/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tsdb/elasticsearch/query_string.go | 28 +++++++++---------- .../tsdb/elasticsearch/query_string_test.go | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index 0c6210a48..0190de54d 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -12,8 +12,9 @@ package elasticsearch import ( "fmt" - parser "github.com/bytedance/go-querystring-parser" elastic "github.com/olivere/elastic/v7" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch/querystring" ) type QueryString struct { @@ -48,10 +49,7 @@ func (s *QueryString) Parser() (elastic.Query, error) { return nil, nil } - // 解析失败，或者没有 nested 字段，则使用透传的方式查询 - //qs := s.queryString(s.q) - - ast, err := parser.Parse(s.q) + ast, err := querystring.Parse(s.q) if err != nil { return nil, err } @@ -81,20 +79,20 @@ func (s *QueryString) check(field string) { } } -func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { +func (s *QueryString) walk(expr querystring.Expr) (elastic.Query, error) { var ( leftQ elastic.Query rightQ elastic.Query err error ) - switch c := condition.(type) { - case *parser.NotCondition: - leftQ, err = s.walk(c.Condition) + switch c := expr.(type) { + case *querystring.NotExpr: + leftQ, err = s.walk(c.Expr) if err != nil { return nil, err } leftQ = elastic.NewBoolQuery().MustNot(leftQ) - case *parser.OrCondition: + case *querystring.OrExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -104,7 +102,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Should(leftQ, rightQ) - case *parser.AndCondition: + case *querystring.AndExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -114,14 +112,14 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Must(leftQ, rightQ) - case *parser.MatchCondition: + case *querystring.MatchExpr: if c.Field != "" { leftQ = elastic.NewMatchPhraseQuery(c.Field, c.Value) s.check(c.Field) } else { leftQ = s.queryString(fmt.Sprintf(`"%s"`, c.Value)) } - case *parser.NumberRangeCondition: + case *querystring.NumberRangeExpr: q := elastic.NewRangeQuery(c.Field) if c.IncludeStart { q.Gte(*c.Start) @@ -135,7 +133,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { } s.check(c.Field) leftQ = q - case *parser.WildcardCondition: + case *querystring.WildcardExpr: if c.Field != "" { leftQ = elastic.NewWildcardQuery(c.Field, c.Value) s.check(c.Field) @@ -143,7 +141,7 @@ func (s *QueryString) walk(condition parser.Condition) (elastic.Query, error) { leftQ = s.queryString(c.Value) } default: - err = fmt.Errorf("condition type is not match %T", condition) + err = fmt.Errorf("expr type is not match %T", expr) } return leftQ, err } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 5687431ff..ea917225f 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -52,7 +52,7 @@ func TestQsToDsl(t *testing.T) { }, { q: `sync_spaces AND -keyword AND -BKLOGAPI`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, + expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"sync_spaces\""}},{"bool":{"must":[{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"keyword\""}}}},{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"BKLOGAPI\""}}}}]}}]}}`, }, { q: `*`, From bc2bb49fb13449852c196b61ad8eaa9dd317c368 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Sat, 8 Feb 2025 21:36:44 +0800 Subject: [PATCH 008/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/query_string.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index 0190de54d..95473201b 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -59,11 +59,6 @@ func (s *QueryString) Parser() (elastic.Query, error) { return nil, err } - // 如果 nestedFields 不存在则直接使用 queryString 透传 - //if len(s.nestedFields) == 0 { - // return qs, nil - //} - for nestedKey := range s.nestedFields { conditionQuery = elastic.NewNestedQuery(nestedKey, conditionQuery) } From 9e6770290b402dd4afa7dfbbe83e89e1d2e0432b Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 10 Feb 2025 19:40:20 +0800 Subject: [PATCH 009/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../querystring/expr.go | 0 .../querystring/lex.go | 0 .../querystring/parser.go | 0 .../querystring/parser_test.go | 0 .../querystring/querystring.y | 0 .../querystring/querystring.y.go | 4 ++-- .../querystring/y.output | 0 .../tsdb/elasticsearch/query_string.go | 18 +++++++++--------- 8 files changed, 11 insertions(+), 11 deletions(-) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/expr.go (100%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/lex.go (100%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/parser.go (100%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/parser_test.go (100%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/querystring.y (100%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/querystring.y.go (99%) rename pkg/unify-query/{tsdb/elasticsearch => internal}/querystring/y.output (100%) diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/expr.go b/pkg/unify-query/internal/querystring/expr.go similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/expr.go rename to pkg/unify-query/internal/querystring/expr.go diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/lex.go b/pkg/unify-query/internal/querystring/lex.go similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/lex.go rename to pkg/unify-query/internal/querystring/lex.go diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/parser.go b/pkg/unify-query/internal/querystring/parser.go similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/parser.go rename to pkg/unify-query/internal/querystring/parser.go diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go b/pkg/unify-query/internal/querystring/parser_test.go similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/parser_test.go rename to pkg/unify-query/internal/querystring/parser_test.go diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y b/pkg/unify-query/internal/querystring/querystring.y similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y rename to pkg/unify-query/internal/querystring/querystring.y diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go b/pkg/unify-query/internal/querystring/querystring.y.go similarity index 99% rename from pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go rename to pkg/unify-query/internal/querystring/querystring.y.go index 27d629a3b..6875aa748 100644 --- a/pkg/unify-query/tsdb/elasticsearch/querystring/querystring.y.go +++ b/pkg/unify-query/internal/querystring/querystring.y.go @@ -11,8 +11,8 @@ import __yyfmt__ "fmt" type yySymType struct { yys int s string - n int - e Expr + n int + e Expr } const tSTRING = 57346 diff --git a/pkg/unify-query/tsdb/elasticsearch/querystring/y.output b/pkg/unify-query/internal/querystring/y.output similarity index 100% rename from pkg/unify-query/tsdb/elasticsearch/querystring/y.output rename to pkg/unify-query/internal/querystring/y.output diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index 95473201b..f8674c750 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -14,7 +14,7 @@ import ( elastic "github.com/olivere/elastic/v7" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch/querystring" + qs "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" ) type QueryString struct { @@ -49,7 +49,7 @@ func (s *QueryString) Parser() (elastic.Query, error) { return nil, nil } - ast, err := querystring.Parse(s.q) + ast, err := qs.Parse(s.q) if err != nil { return nil, err } @@ -74,20 +74,20 @@ func (s *QueryString) check(field string) { } } -func (s *QueryString) walk(expr querystring.Expr) (elastic.Query, error) { +func (s *QueryString) walk(expr qs.Expr) (elastic.Query, error) { var ( leftQ elastic.Query rightQ elastic.Query err error ) switch c := expr.(type) { - case *querystring.NotExpr: + case *qs.NotExpr: leftQ, err = s.walk(c.Expr) if err != nil { return nil, err } leftQ = elastic.NewBoolQuery().MustNot(leftQ) - case *querystring.OrExpr: + case *qs.OrExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -97,7 +97,7 @@ func (s *QueryString) walk(expr querystring.Expr) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Should(leftQ, rightQ) - case *querystring.AndExpr: + case *qs.AndExpr: leftQ, err = s.walk(c.Left) if err != nil { return nil, err @@ -107,14 +107,14 @@ func (s *QueryString) walk(expr querystring.Expr) (elastic.Query, error) { return nil, err } leftQ = elastic.NewBoolQuery().Must(leftQ, rightQ) - case *querystring.MatchExpr: + case *qs.MatchExpr: if c.Field != "" { leftQ = elastic.NewMatchPhraseQuery(c.Field, c.Value) s.check(c.Field) } else { leftQ = s.queryString(fmt.Sprintf(`"%s"`, c.Value)) } - case *querystring.NumberRangeExpr: + case *qs.NumberRangeExpr: q := elastic.NewRangeQuery(c.Field) if c.IncludeStart { q.Gte(*c.Start) @@ -128,7 +128,7 @@ func (s *QueryString) walk(expr querystring.Expr) (elastic.Query, error) { } s.check(c.Field) leftQ = q - case *querystring.WildcardExpr: + case *qs.WildcardExpr: if c.Field != "" { leftQ = elastic.NewWildcardQuery(c.Field, c.Value) s.check(c.Field) From 9c8bf1484cd9f43aab986b951095204a33af1264 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 10 Feb 2025 19:49:28 +0800 Subject: [PATCH 010/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/querystring/lex.go | 11 ++++------- pkg/unify-query/internal/querystring/parser.go | 2 -- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/unify-query/internal/querystring/lex.go b/pkg/unify-query/internal/querystring/lex.go index ada6eef07..3f62e1bb1 100644 --- a/pkg/unify-query/internal/querystring/lex.go +++ b/pkg/unify-query/internal/querystring/lex.go @@ -17,12 +17,9 @@ package querystring import ( "bufio" "bytes" - "context" "io" "strings" "unicode" - - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) const reservedChars = "+-=&|> Date: Tue, 11 Feb 2025 11:21:38 +0800 Subject: [PATCH 011/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/instance.go | 2 +- pkg/unify-query/tsdb/elasticsearch/query_string.go | 11 ++++++++--- .../tsdb/elasticsearch/query_string_test.go | 14 +++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index a0a94ad28..80b28b4a2 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -233,7 +233,7 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac // querystring 生成 elastic.query if qb.QueryString != "" { qs := NewQueryString(qb.QueryString, fact.NestedField) - q, qsErr := qs.Parser() + q, qsErr := qs.ToDSL() if qsErr != nil { return nil, qsErr } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index f8674c750..b9cd4b05f 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -44,19 +44,24 @@ func (s *QueryString) queryString(str string) elastic.Query { return elastic.NewQueryStringQuery(str).AnalyzeWildcard(true) } -func (s *QueryString) Parser() (elastic.Query, error) { +func (s *QueryString) ToDSL() (elastic.Query, error) { if s.q == "" || s.q == "*" { return nil, nil } + q := s.queryString(s.q) ast, err := qs.Parse(s.q) if err != nil { - return nil, err + return q, nil } conditionQuery, err := s.walk(ast) if err != nil { - return nil, err + return q, nil + } + + if len(s.nestedFields) == 0 { + return q, nil } for nestedKey := range s.nestedFields { diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index ea917225f..1a5ddb278 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -31,15 +31,18 @@ func TestQsToDsl(t *testing.T) { err error }{ { - q: `log: "ERROR MSG"`, + q: `log: "ERROR MSG"`, + //expected: `{"query_string":{"analyze_wildcard":true,"query":"log: \"ERROR MSG\""}}` expected: `{"match_phrase":{"log":{"query":"ERROR MSG"}}}`, }, { - q: `quick brown fox`, + q: `quick brown fox`, + //expected: `{"query_string":{"analyze_wildcard":true,"query":"quick brown fox"}}`, expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"quick\""}},{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"brown\""}},{"query_string":{"analyze_wildcard":true,"query":"\"fox\""}}]}}]}}`, }, { - q: `word.key: qu?ck`, + q: `word.key: qu?ck`, + //expected: `{"query_string":{"analyze_wildcard":true,"query":"word.key: qu?ck"}}` expected: `{"wildcard":{"word.key":{"value":"qu?ck"}}}`, }, { @@ -51,7 +54,8 @@ func TestQsToDsl(t *testing.T) { expected: `{"nested":{"path":"nested.key","query":{"bool":{"must":[{"match_phrase":{"nested.key":{"query":"test"}}},{"query_string":{"analyze_wildcard":true,"query":"\"demo\""}}]}}}}`, }, { - q: `sync_spaces AND -keyword AND -BKLOGAPI`, + q: `sync_spaces AND -keyword AND -BKLOGAPI`, + //expected: `{"query_string":{"analyze_wildcard":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"sync_spaces\""}},{"bool":{"must":[{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"keyword\""}}}},{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"BKLOGAPI\""}}}}]}}]}}`, }, { @@ -69,7 +73,7 @@ func TestQsToDsl(t *testing.T) { } return "" }) - query, err := qs.Parser() + query, err := qs.ToDSL() if err == nil { if query != nil { body, _ := query.Source() From 9ca61bf9d45c847b13948fe2f8247c56a5a1b8f9 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 11 Feb 2025 14:45:41 +0800 Subject: [PATCH 012/104] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20es=20query?= =?UTF-8?q?=20string=20=E8=AF=AD=E6=B3=95=E8=A7=A3=E6=9E=90=20#10101580811?= =?UTF-8?q?21859811?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/querystring/parser_test.go | 30 +++++++++++++++++-- .../internal/querystring/querystring.y | 2 +- .../internal/querystring/querystring.y.go | 6 ++-- .../tsdb/elasticsearch/query_string.go | 2 ++ .../tsdb/elasticsearch/query_string_test.go | 24 +++++++-------- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/pkg/unify-query/internal/querystring/parser_test.go b/pkg/unify-query/internal/querystring/parser_test.go index 659fb2dcb..fa7187264 100644 --- a/pkg/unify-query/internal/querystring/parser_test.go +++ b/pkg/unify-query/internal/querystring/parser_test.go @@ -69,7 +69,7 @@ func TestParser(t *testing.T) { }, "通配符匹配": { q: `qu?ck bro*`, - e: &AndExpr{ + e: &OrExpr{ Left: &WildcardExpr{ Value: "qu?ck", }, @@ -185,13 +185,37 @@ func TestParser(t *testing.T) { }, }, }, + "new-1": { + q: `quick brown +fox -news`, + e: &OrExpr{ + Left: &MatchExpr{ + Value: "quick", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Value: "brown", + }, + Right: &OrExpr{ + Left: &MatchExpr{ + Value: "fox", + }, + Right: &NotExpr{ + Expr: &MatchExpr{ + Field: "", + Value: "news", + }, + }, + }, + }, + }, + }, "模糊匹配": { q: `quick brown fox`, - e: &AndExpr{ + e: &OrExpr{ Left: &MatchExpr{ Value: "quick", }, - Right: &AndExpr{ + Right: &OrExpr{ Left: &MatchExpr{ Value: "brown", }, diff --git a/pkg/unify-query/internal/querystring/querystring.y b/pkg/unify-query/internal/querystring/querystring.y index 26a9aadfd..377925eaa 100644 --- a/pkg/unify-query/internal/querystring/querystring.y +++ b/pkg/unify-query/internal/querystring/querystring.y @@ -35,7 +35,7 @@ searchLogicParts { searchLogicParts: searchLogicPart searchLogicParts { - $$ = NewAndExpr($1, $2) + $$ = NewOrExpr($1, $2) } | searchLogicPart { diff --git a/pkg/unify-query/internal/querystring/querystring.y.go b/pkg/unify-query/internal/querystring/querystring.y.go index 6875aa748..86e619b3c 100644 --- a/pkg/unify-query/internal/querystring/querystring.y.go +++ b/pkg/unify-query/internal/querystring/querystring.y.go @@ -11,8 +11,8 @@ import __yyfmt__ "fmt" type yySymType struct { yys int s string - n int - e Expr + n int + e Expr } const tSTRING = 57346 @@ -508,7 +508,7 @@ yydefault: yyDollar = yyS[yypt-2 : yypt+1] //line querystring.y:37 { - yyVAL.e = NewAndExpr(yyDollar[1].e, yyDollar[2].e) + yyVAL.e = NewOrExpr(yyDollar[1].e, yyDollar[2].e) } case 3: yyDollar = yyS[yypt-1 : yypt+1] diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index b9cd4b05f..e9ba5f390 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -49,6 +49,7 @@ func (s *QueryString) ToDSL() (elastic.Query, error) { return nil, nil } + // 解析失败，或者没有 nested 字段，则使用透传的方式查询 q := s.queryString(s.q) ast, err := qs.Parse(s.q) if err != nil { @@ -60,6 +61,7 @@ func (s *QueryString) ToDSL() (elastic.Query, error) { return q, nil } + // 如果 nestedFields 不存在则直接使用 queryString 透传 if len(s.nestedFields) == 0 { return q, nil } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 1a5ddb278..3f8e50f6b 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -31,19 +31,19 @@ func TestQsToDsl(t *testing.T) { err error }{ { - q: `log: "ERROR MSG"`, - //expected: `{"query_string":{"analyze_wildcard":true,"query":"log: \"ERROR MSG\""}}` - expected: `{"match_phrase":{"log":{"query":"ERROR MSG"}}}`, + q: `log: "ERROR MSG"`, + expected: `{"query_string":{"analyze_wildcard":true,"query":"log: \"ERROR MSG\""}}`, + //expected: `{"match_phrase":{"log":{"query":"ERROR MSG"}}}`, }, { - q: `quick brown fox`, - //expected: `{"query_string":{"analyze_wildcard":true,"query":"quick brown fox"}}`, - expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"quick\""}},{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"brown\""}},{"query_string":{"analyze_wildcard":true,"query":"\"fox\""}}]}}]}}`, + q: `quick brown fox`, + expected: `{"query_string":{"analyze_wildcard":true,"query":"quick brown fox"}}`, + //expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"quick\""}},{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"brown\""}},{"query_string":{"analyze_wildcard":true,"query":"\"fox\""}}]}}]}}`, }, { - q: `word.key: qu?ck`, - //expected: `{"query_string":{"analyze_wildcard":true,"query":"word.key: qu?ck"}}` - expected: `{"wildcard":{"word.key":{"value":"qu?ck"}}}`, + q: `word.key: qu?ck`, + expected: `{"query_string":{"analyze_wildcard":true,"query":"word.key: qu?ck"}}`, + //expected: `{"wildcard":{"word.key":{"value":"qu?ck"}}}`, }, { q: "\"message queue conflict\"", @@ -54,9 +54,9 @@ func TestQsToDsl(t *testing.T) { expected: `{"nested":{"path":"nested.key","query":{"bool":{"must":[{"match_phrase":{"nested.key":{"query":"test"}}},{"query_string":{"analyze_wildcard":true,"query":"\"demo\""}}]}}}}`, }, { - q: `sync_spaces AND -keyword AND -BKLOGAPI`, - //expected: `{"query_string":{"analyze_wildcard":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, - expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"sync_spaces\""}},{"bool":{"must":[{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"keyword\""}}}},{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"BKLOGAPI\""}}}}]}}]}}`, + q: `sync_spaces AND -keyword AND -BKLOGAPI`, + expected: `{"query_string":{"analyze_wildcard":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, + //expected: `{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"\"sync_spaces\""}},{"bool":{"must":[{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"keyword\""}}}},{"bool":{"must_not":{"query_string":{"analyze_wildcard":true,"query":"\"BKLOGAPI\""}}}}]}}]}}`, }, { q: `*`, From 6093924fc78be15ed96a1eac5f4c02b57cb16164 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Feb 2025 16:40:03 +0800 Subject: [PATCH 013/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/query_ts.go | 20 +++++ pkg/unify-query/service/http/query.go | 91 +++++++++++++------- 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index bd02d3a1a..bd91340ea 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -564,6 +564,26 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q MetricName: metricName, Aggregates: aggregates, BkSqlCondition: allConditions.BkSql(), + Size: q.Limit, + From: q.From, + } + + if len(q.OrderBy) > 0 { + qry.Orders = make(metadata.Orders) + for _, o := range q.OrderBy { + if len(o) == 0 { + continue + } + + asc := true + name := o + + if strings.HasPrefix(o, "-") { + asc = false + name = name[1:] + } + qry.Orders[name] = asc + } } metadata.GetQueryParams(ctx).SetStorageType(qry.StorageType) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 7e9a776f9..e3f6063ea 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -285,11 +285,11 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot return } -func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs) (*PromData, error) { +func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.QueryTs) (*PromData, error) { var ( res any err error - resp = NewPromData(query.ResultColumns) + resp = NewPromData(queryTs.ResultColumns) ) ctx, span := trace.NewSpan(ctx, "query-reference") @@ -299,35 +299,52 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs span.End(&err) }() - qStr, _ := json.Marshal(query) + qStr, _ := json.Marshal(queryTs) span.Set("query-ts", string(qStr)) - for _, q := range query.QueryList { - q.IsReference = true - if q.TableID == "" { + for _, ql := range queryTs.QueryList { + ql.IsReference = true + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } + + if ql.TableID == "" { err = fmt.Errorf("tableID is empty") return nil, err } } // 判断如果 step 为空，则补充默认 step - if query.Step == "" { - query.Step = promql.GetDefaultStep().String() + if queryTs.Step == "" { + queryTs.Step = promql.GetDefaultStep().String() } - queryRef, err := query.ToQueryReference(ctx) - startInt, err := strconv.ParseInt(query.Start, 10, 64) + queryRef, err := queryTs.ToQueryReference(ctx) + startInt, err := strconv.ParseInt(queryTs.Start, 10, 64) if err != nil { return nil, err } start := time.Unix(startInt, 0) - endInt, err := strconv.ParseInt(query.End, 10, 64) + endInt, err := strconv.ParseInt(queryTs.End, 10, 64) if err != nil { return nil, err } end := time.Unix(endInt, 0) - step, err := model.ParseDuration(query.Step) + step, err := model.ParseDuration(queryTs.Step) if err != nil { return nil, err } @@ -337,8 +354,8 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs metadata.SetQueryReference(ctx, queryRef) var lookBackDelta time.Duration - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return nil, err } @@ -349,10 +366,10 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs Timeout: SingleflightTimeout, }, lookBackDelta, QueryMaxRouting) - if query.Instant { - res, err = instance.DirectQuery(ctx, query.MetricMerge, start) + if queryTs.Instant { + res, err = instance.DirectQuery(ctx, queryTs.MetricMerge, start) } else { - res, err = instance.DirectQueryRange(ctx, query.MetricMerge, start, end, time.Duration(step)) + res, err = instance.DirectQueryRange(ctx, queryTs.MetricMerge, start, end, time.Duration(step)) } if err != nil { return nil, err @@ -397,7 +414,7 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs } // queryTsToInstanceAndStmt query 结构体转换为 instance 以及 stmt -func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { +func queryTsToInstanceAndStmt(ctx context.Context, queryTs *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { var ( lookBackDelta time.Duration promExprOpt = &structured.PromExprOption{} @@ -408,37 +425,53 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i span.End(&err) }() - queryString, _ := json.Marshal(query) + queryString, _ := json.Marshal(queryTs) span.Set("query-ts", queryString) // 限制 queryList 是否过长 if DefaultQueryListLimit > 0 { - if len(query.QueryList) > DefaultQueryListLimit { + if len(queryTs.QueryList) > DefaultQueryListLimit { err = fmt.Errorf("the number of query lists cannot be greater than %d", DefaultQueryListLimit) } } // 判断是否打开对齐 - for _, q := range query.QueryList { - q.IsReference = false - q.AlignInfluxdbResult = AlignInfluxdbResult + for _, ql := range queryTs.QueryList { + ql.IsReference = false + ql.AlignInfluxdbResult = AlignInfluxdbResult + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } } // 判断是否指定 LookBackDelta - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return } } // 如果 step 为空，则补充默认 step - if query.Step == "" { - query.Step = promql.GetDefaultStep().String() + if queryTs.Step == "" { + queryTs.Step = promql.GetDefaultStep().String() } // 转换成 queryRef - queryRef, err := query.ToQueryReference(ctx) + queryRef, err := queryTs.ToQueryReference(ctx) if err != nil { return } @@ -467,7 +500,7 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i }, lookBackDelta, QueryMaxRouting) } - expr, err := query.ToPromExpr(ctx, promExprOpt) + expr, err := queryTs.ToPromExpr(ctx, promExprOpt) if err != nil { return } From bd07b1182bba74d437faa5c49130e0aba7ef491f Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Feb 2025 17:35:14 +0800 Subject: [PATCH 014/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance.go | 80 ++++++++++++++++++- .../tsdb/elasticsearch/instance.go | 2 +- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 28e6d0827..91a87a442 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -31,6 +31,14 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" ) +const ( + KeyHighLight = "__highlight" + + KeyIndex = "__index" + KeyTableID = "__result_table" + KeyDataLabel = "__data_label" +) + type Instance struct { ctx context.Context @@ -275,15 +283,81 @@ func (i *Instance) table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, err error) { + + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("doris query error: %s", r) + } + }() + + ctx, span := trace.NewSpan(ctx, "bk-sql-query-raw") + defer span.End(&err) + + span.Set("query-raw-start", start) + span.Set("query-raw-end", end) + + if start.UnixMilli() > end.UnixMilli() || start.UnixMilli() == 0 { + return + } + + rangeLeftTime := end.Sub(start) + metric.TsDBRequestRangeMinute(ctx, rangeLeftTime, i.InstanceType()) + + if i.maxLimit > 0 { + maxLimit := i.maxLimit + i.tolerance + // 如果不传 size，则取最大的限制值 + if query.Size == 0 || query.Size > i.maxLimit { + query.Size = maxLimit + } + } + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + + sql, err := queryFactory.SQL() + if err != nil { + return + } + + data, err := i.sqlQuery(ctx, sql, span) + if err != nil { + return + } + + if data == nil { + return + } + + span.Set("data-total-records", data.TotalRecords) + log.Infof(ctx, "total records: %d", data.TotalRecords) + + if i.maxLimit > 0 && data.TotalRecords > i.maxLimit { + return + } + + for _, list := range data.List { + + list[KeyIndex] = query.DB + list[KeyTableID] = query.TableID + list[KeyDataLabel] = query.DataLabel + list[FieldTime] = list[timeStamp] + + if query.HighLight.Enable { + list[KeyHighLight] = "" + } + + dataCh <- list + } + + total = int64(data.TotalRecords) + return } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { var ( err error ) - ctx, span := trace.NewSpan(ctx, "bk-sql-raw") + ctx, span := trace.NewSpan(ctx, "bk-sql-query-series-set") defer span.End(&err) span.Set("query-series-set-start", start) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index a0a94ad28..14307f410 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -626,7 +626,7 @@ func (i *Instance) QuerySeriesSet( err error ) - ctx, span := trace.NewSpan(ctx, "elasticsearch-query-reference") + ctx, span := trace.NewSpan(ctx, "elasticsearch-query-series-set") defer span.End(&err) if len(query.Aggregates) == 0 { From b84789b1395c29bccfa9eb2ffdeaa9b0d25e1a6c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 18 Feb 2025 22:29:05 +0800 Subject: [PATCH 015/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/query_ts.go | 12 +++++----- pkg/unify-query/service/influxdb/hook.go | 3 +++ pkg/unify-query/service/influxdb/service.go | 5 +---- pkg/unify-query/service/influxdb/settings.go | 4 ++++ pkg/unify-query/tsdb/bksql/instance.go | 23 +++++--------------- 5 files changed, 19 insertions(+), 28 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index bd91340ea..8df8cc057 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -892,21 +892,19 @@ func (q *Query) BuildMetadataQuery( return nil, err } - // 在 metadata 还没有补充 storageType 字段之前 - // 使用 sourceType 来判断是否是 es 查询 - // 等后面支持了之后可以删除该段逻辑 - if tsDB.SourceType == BkData { - query.StorageType = consul.ElasticsearchStorageType - } - // 兼容原逻辑，storageType 通过 storageMap 获取 if query.StorageType == "" { + log.Warnf(ctx, "storageType is empty with %s", query.TableID) stg, _ := tsdb.GetStorage(query.StorageID) if stg != nil { query.StorageType = stg.Type } } + if query.StorageType == "" { + return nil, fmt.Errorf("storageType is empty with %d", query.StorageID) + } + query.Measurement = measurement query.Measurements = measurements query.Field = field diff --git a/pkg/unify-query/service/influxdb/hook.go b/pkg/unify-query/service/influxdb/hook.go index fa4521b7a..ec681fad3 100644 --- a/pkg/unify-query/service/influxdb/hook.go +++ b/pkg/unify-query/service/influxdb/hook.go @@ -45,6 +45,8 @@ func setDefaultConfig() { viper.SetDefault(GrpcMaxCallRecvMsgSizeConfigPath, 1024*1024*10) viper.SetDefault(GrpcMaxCallSendMsgSizeConfigPath, 1024*1024*10) + + viper.SetDefault(IsCacheConfigPath, true) } // LoadConfig @@ -71,6 +73,7 @@ func LoadConfig() { SpaceRouterBboltPath = viper.GetString(SpaceRouterBboltPathConfigPath) SpaceRouterBboltBucketName = viper.GetString(SpaceRouterBboltBucketNameConfigPath) SpaceRouterBboltWriteBatchSize = viper.GetInt(SpaceRouterBboltWriteBatchSizeConfigPath) + IsCache = viper.GetBool(IsCacheConfigPath) GrpcMaxCallRecvMsgSize = viper.GetInt(GrpcMaxCallRecvMsgSizeConfigPath) GrpcMaxCallSendMsgSize = viper.GetInt(GrpcMaxCallSendMsgSizeConfigPath) diff --git a/pkg/unify-query/service/influxdb/service.go b/pkg/unify-query/service/influxdb/service.go index f8b41cf12..39bffbf4d 100644 --- a/pkg/unify-query/service/influxdb/service.go +++ b/pkg/unify-query/service/influxdb/service.go @@ -69,19 +69,16 @@ func (s *Service) Reload(ctx context.Context) { err = s.loopReloadStorage(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload es storage failed for->[%s]", err) - return } err = s.loopReloadTableInfo(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload table info failed,error:%s", err) - return } err = s.loopReloadRouter(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload query router failed,error:%s", err) - return } err = s.loopReloadBCSInfo(s.ctx) @@ -313,7 +310,7 @@ func (s *Service) reloadInfluxDBRouter(ctx context.Context) error { // reloadInfluxDBRouter 重新加载 SpaceTsDbRouter func (s *Service) reloadSpaceTsDbRouter(ctx context.Context) error { - ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, true) + ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, IsCache) if err != nil { return err } diff --git a/pkg/unify-query/service/influxdb/settings.go b/pkg/unify-query/service/influxdb/settings.go index 95fe57921..430c7dd15 100644 --- a/pkg/unify-query/service/influxdb/settings.go +++ b/pkg/unify-query/service/influxdb/settings.go @@ -37,6 +37,8 @@ const ( GrpcMaxCallRecvMsgSizeConfigPath = "influxdb.grpc_max_call_recv_msg_size" GrpcMaxCallSendMsgSizeConfigPath = "influxdb.grpc_max_call_send_msg_size" + + IsCacheConfigPath = "influxdb.is_cache" ) var ( @@ -61,6 +63,8 @@ var ( SpaceRouterBboltBucketName string SpaceRouterBboltWriteBatchSize int + IsCache bool + GrpcMaxCallRecvMsgSize int GrpcMaxCallSendMsgSize int ) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 91a87a442..d67721292 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -56,11 +56,6 @@ type Instance struct { client *Client } -func (i *Instance) QueryReference(ctx context.Context, query *metadata.Query, start int64, end int64) (*prompb.QueryResult, error) { - //TODO implement me - panic("implement me") -} - var _ tsdb.Instance = (*Instance)(nil) type Options struct { @@ -410,18 +405,15 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } func (i *Instance) DirectQueryRange(ctx context.Context, promql string, start, end time.Time, step time.Duration) (promql.Matrix, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectQuery(ctx context.Context, qs string, end time.Time) (promql.Vector, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, start, end time.Time) ([]string, error) { @@ -492,18 +484,15 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, } func (i *Instance) QuerySeries(ctx context.Context, query *metadata.Query, start, end time.Time) ([]map[string]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelNames(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelValues(ctx context.Context, name string, start, end time.Time, limit int, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) InstanceType() string { From 6dff927c3245b9f3336d306e1009307841aa0995 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 20 Feb 2025 10:49:04 +0800 Subject: [PATCH 016/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 13 ++ pkg/unify-query/tsdb/bksql/instance.go | 2 - pkg/unify-query/tsdb/bksql/querystring.go | 140 ++++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 pkg/unify-query/tsdb/bksql/querystring.go diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 04433502c..b0d85c3d5 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -191,6 +191,19 @@ func (f *QueryFactory) SQL() (sql string, err error) { f.write("AND") f.write("(" + f.query.BkSqlCondition + ")") } + + // 拼接 querystring 数据 + if f.query.QueryString != "" { + qsSql, qsErr := QueryStringToSQL(f.query.QueryString) + if qsErr != nil { + err = qsErr + return + } + + f.write("AND") + f.write(qsSql) + } + if len(f.groups) > 0 { f.write("GROUP BY") f.write(strings.Join(f.groups, ", ")) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index d67721292..91867371a 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -279,7 +279,6 @@ func (i *Instance) table(query *metadata.Query) string { // QueryRawData 直接查询原始返回 func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, err error) { - defer func() { if r := recover(); r != nil { err = fmt.Errorf("doris query error: %s", r) @@ -331,7 +330,6 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } for _, list := range data.List { - list[KeyIndex] = query.DB list[KeyTableID] = query.TableID list[KeyDataLabel] = query.DataLabel diff --git a/pkg/unify-query/tsdb/bksql/querystring.go b/pkg/unify-query/tsdb/bksql/querystring.go new file mode 100644 index 000000000..a4da2950a --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/querystring.go @@ -0,0 +1,140 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package bksql + +import ( + "fmt" + "strings" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" +) + +var ( + ErrorMatchAll = "doris 不支持全字段检索" +) + +func QueryStringToSQL(q string) (string, error) { + e, err := querystring.Parse(q) + if err != nil { + return "", err + } + + p := sqlParser{ + e: e, + } + + return parser(p) +} + +type sqlParser struct { + e querystring.Expr + not bool +} + +func parser(p sqlParser) (string, error) { + var ( + err error + left querystring.Expr + right querystring.Expr + ) + + switch c := p.e.(type) { + case *querystring.NotExpr: + p.not = !p.not + p.e = c.Expr + return parser(p) + case *querystring.OrExpr: + left, err = parser(sqlParser{not: p.not, e: c.Left}) + if err != nil { + return "", err + } + right, err = parser(sqlParser{not: p.not, e: c.Right}) + if err != nil { + return "", err + } + return fmt.Sprintf("( %s OR %s)", left, right), nil + case *querystring.AndExpr: + left, err = parser(sqlParser{not: p.not, e: c.Left}) + if err != nil { + return "", err + } + right, err = parser(sqlParser{not: p.not, e: c.Right}) + if err != nil { + return "", err + } + return fmt.Sprintf("( %s AND %s)", left, right), nil + case *querystring.MatchExpr: + if c.Field == "" { + err = fmt.Errorf(ErrorMatchAll + ": " + c.Value) + return "", err + } + + if p.not { + return fmt.Sprintf("%s NOT LIKE '%%%s%%'", c.Field, c.Value), nil + } else { + return fmt.Sprintf("%s LIKE '%%%s%%'", c.Field, c.Value), nil + } + case *querystring.NumberRangeExpr: + if c.Field == "" { + err = fmt.Errorf(ErrorMatchAll) + return "", err + } + + var ( + start string + end string + ) + if *c.Start != "*" { + var op string + if !p.not { + if c.IncludeStart { + op = ">=" + } else { + op = ">" + } + } else { + if c.IncludeStart { + op = "<" + } else { + op = "<=" + } + } + start = fmt.Sprintf("%s %s %s", c.Field, op, *c.Start) + } + + if *c.End != "*" { + var op string + if !p.not { + if c.IncludeEnd { + op = "<=" + } else { + op = "<" + } + } else { + if c.IncludeEnd { + op = ">" + } else { + op = ">=" + } + } + end = fmt.Sprintf("%s %s %s", c.Field, op, *c.End) + } + + if p.not { + return fmt.Sprintf("NOT ( %s )", strings.Join([]string{start, end}, " AND ")), nil + } else { + return fmt.Sprintf("( %s )", strings.Join([]string{start, end}, " AND ")), nil + } + default: + err = fmt.Errorf("expr type is not match %T", p.e) + } + + return "", err +} From 73302fcd3dc9ef10b29dcb8b18982ea420a37aa6 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 21 Feb 2025 15:43:42 +0800 Subject: [PATCH 017/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/querystring/expr.go | 5 + pkg/unify-query/tsdb/bksql/format.go | 4 +- pkg/unify-query/tsdb/bksql/querystring.go | 142 +++++++++++------- .../tsdb/bksql/querystring_test.go | 105 +++++++++++++ 4 files changed, 199 insertions(+), 57 deletions(-) create mode 100644 pkg/unify-query/tsdb/bksql/querystring_test.go diff --git a/pkg/unify-query/internal/querystring/expr.go b/pkg/unify-query/internal/querystring/expr.go index b9e8ea175..c070dbbc7 100644 --- a/pkg/unify-query/internal/querystring/expr.go +++ b/pkg/unify-query/internal/querystring/expr.go @@ -172,3 +172,8 @@ func newStringExpr(str string) FieldableExpr { return NewMatchExpr(str) } + +type TermExpr struct { + Field string + Value string +} diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index b0d85c3d5..5edbbbb9f 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -194,7 +194,9 @@ func (f *QueryFactory) SQL() (sql string, err error) { // 拼接 querystring 数据 if f.query.QueryString != "" { - qsSql, qsErr := QueryStringToSQL(f.query.QueryString) + // 字段类型 + fieldsMap := make(map[string]string) + qsSql, qsErr := NewDorisSQLExpr(f.query.QueryString).WithFieldsMap(fieldsMap).Parser() if qsErr != nil { err = qsErr return diff --git a/pkg/unify-query/tsdb/bksql/querystring.go b/pkg/unify-query/tsdb/bksql/querystring.go index a4da2950a..e3ea175ae 100644 --- a/pkg/unify-query/tsdb/bksql/querystring.go +++ b/pkg/unify-query/tsdb/bksql/querystring.go @@ -16,71 +16,121 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" ) +const ( + KeyWord = "keyword" + Text = "text" + Integer = "integer" + Long = "long" + Date = "date" +) + var ( ErrorMatchAll = "doris 不支持全字段检索" ) -func QueryStringToSQL(q string) (string, error) { - e, err := querystring.Parse(q) +type DorisSQLExpr struct { + qs string + fieldsMap map[string]string +} + +func NewDorisSQLExpr(qs string) *DorisSQLExpr { + return &DorisSQLExpr{ + qs: qs, + } +} + +func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) *DorisSQLExpr { + s.fieldsMap = fieldsMap + return s +} + +func (s *DorisSQLExpr) checkMatchALL(k string) bool { + if s.fieldsMap != nil { + if t, ok := s.fieldsMap[k]; ok { + if t == Text { + return true + } + } + } + return false +} + +func (s *DorisSQLExpr) Parser() (string, error) { + expr, err := querystring.Parse(s.qs) if err != nil { return "", err } - - p := sqlParser{ - e: e, + if expr == nil { + return "", nil } - return parser(p) + return s.walk(expr) } -type sqlParser struct { - e querystring.Expr - not bool +func (s *DorisSQLExpr) fieldFormat(field string) string { + fs := strings.Split(field, ".") + if len(fs) > 1 { + return fmt.Sprintf(`CAST(%s["%s"] AS STRING)`, fs[0], strings.Join(fs[1:], `"]["`)) + } + return fmt.Sprintf("`%s`", field) } -func parser(p sqlParser) (string, error) { +func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { var ( err error - left querystring.Expr - right querystring.Expr + left string + right string ) - switch c := p.e.(type) { + if s == nil { + return "", nil + } + + switch c := e.(type) { case *querystring.NotExpr: - p.not = !p.not - p.e = c.Expr - return parser(p) + left, err = s.walk(c.Expr) + if err != nil { + return "", err + } + return fmt.Sprintf("NOT ( %s )", left), nil case *querystring.OrExpr: - left, err = parser(sqlParser{not: p.not, e: c.Left}) + left, err = s.walk(c.Left) if err != nil { return "", err } - right, err = parser(sqlParser{not: p.not, e: c.Right}) + right, err = s.walk(c.Right) if err != nil { return "", err } - return fmt.Sprintf("( %s OR %s)", left, right), nil + return fmt.Sprintf("( %s OR %s )", left, right), nil case *querystring.AndExpr: - left, err = parser(sqlParser{not: p.not, e: c.Left}) + left, err = s.walk(c.Left) if err != nil { return "", err } - right, err = parser(sqlParser{not: p.not, e: c.Right}) + right, err = s.walk(c.Right) if err != nil { return "", err } - return fmt.Sprintf("( %s AND %s)", left, right), nil + return fmt.Sprintf("( %s AND %s )", left, right), nil + case *querystring.WildcardExpr: + if c.Field == "" { + err = fmt.Errorf(ErrorMatchAll) + return "", err + } + + return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil case *querystring.MatchExpr: if c.Field == "" { err = fmt.Errorf(ErrorMatchAll + ": " + c.Value) return "", err } - if p.not { - return fmt.Sprintf("%s NOT LIKE '%%%s%%'", c.Field, c.Value), nil - } else { - return fmt.Sprintf("%s LIKE '%%%s%%'", c.Field, c.Value), nil + if s.checkMatchALL(c.Field) { + return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil } + + return fmt.Sprintf("%s = '%s'", s.fieldFormat(c.Field), c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { err = fmt.Errorf(ErrorMatchAll) @@ -93,47 +143,27 @@ func parser(p sqlParser) (string, error) { ) if *c.Start != "*" { var op string - if !p.not { - if c.IncludeStart { - op = ">=" - } else { - op = ">" - } + if c.IncludeStart { + op = ">=" } else { - if c.IncludeStart { - op = "<" - } else { - op = "<=" - } + op = ">" } - start = fmt.Sprintf("%s %s %s", c.Field, op, *c.Start) + start = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.Start) } if *c.End != "*" { var op string - if !p.not { - if c.IncludeEnd { - op = "<=" - } else { - op = "<" - } + if c.IncludeEnd { + op = "<=" } else { - if c.IncludeEnd { - op = ">" - } else { - op = ">=" - } + op = "<" } - end = fmt.Sprintf("%s %s %s", c.Field, op, *c.End) + end = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.End) } - if p.not { - return fmt.Sprintf("NOT ( %s )", strings.Join([]string{start, end}, " AND ")), nil - } else { - return fmt.Sprintf("( %s )", strings.Join([]string{start, end}, " AND ")), nil - } + return fmt.Sprintf("( %s )", strings.Join([]string{start, end}, " AND ")), nil default: - err = fmt.Errorf("expr type is not match %T", p.e) + err = fmt.Errorf("expr type is not match %T", e) } return "", err diff --git a/pkg/unify-query/tsdb/bksql/querystring_test.go b/pkg/unify-query/tsdb/bksql/querystring_test.go new file mode 100644 index 000000000..be146637b --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/querystring_test.go @@ -0,0 +1,105 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package bksql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestQueryStringToSQL(t *testing.T) { + tests := []struct { + name string + input string + want string + err string + }{ + { + name: "simple match", + input: "name:test", + want: "`name` = 'test'", + }, + { + name: "one word", + input: "test", + err: "doris 不支持全字段检索: test", + }, + { + name: "complex nested query", + input: "(a:1 AND (b:2 OR c:3)) OR NOT d:4", + want: "( ( `a` = '1' AND ( `b` = '2' OR `c` = '3' ) ) OR NOT ( `d` = '4' ) )", + }, + { + name: "invalid syntax", + input: "name:test AND OR", + err: "syntax error: unexpected tOR", + }, + { + name: "empty input", + input: "", + }, + { + name: "OR expression with multiple terms", + input: "a:1 OR b:2 OR c:3", + want: "( `a` = '1' OR ( `b` = '2' OR `c` = '3' ) )", + }, + { + name: "mixed AND/OR with proper precedence", + input: "a:1 AND b:2 OR c:3", + want: "( `a` = '1' AND ( `b` = '2' OR `c` = '3' ) )", + }, + { + name: "exact match with quotes", + input: "name:\"exact match\"", + want: "`name` = 'exact match'", + }, + { + name: "numeric equality", + input: "age:25", + want: "`age` = '25'", + }, + { + name: "date range query", + input: "timestamp:[2023-01-01 TO 2023-12-31]", + want: "( timestamp >= '2023-01-01' AND timestamp <= '2023-12-31' )", + err: "syntax error: unexpected tSTRING, expecting tPHRASE or tNUMBER or tSTAR or tMINUS", + }, + { + name: "invalid field name", + input: "123field:value", + want: "`123field` = 'value'", + }, + { + name: "text filter", + input: "text:value", + want: "`text` LIKE '%value%'", + }, + { + name: "object field", + input: "__ext.container_name: value", + want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewDorisSQLExpr(tt.input).WithFieldsMap(map[string]string{ + "text": Text, + }).Parser() + if err != nil { + assert.Equal(t, err.Error(), tt.err) + return + } + + assert.Equal(t, tt.want, got) + }) + } +} From d837422460f8ac6ea930a178589b83dc1ff99a9d Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 10:35:21 +0800 Subject: [PATCH 018/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + pkg/unify-query/metadata/struct.go | 21 +- pkg/unify-query/tsdb/bksql/client_test.go | 19 +- pkg/unify-query/tsdb/bksql/format.go | 74 ++++-- pkg/unify-query/tsdb/bksql/format_test.go | 56 ++-- pkg/unify-query/tsdb/bksql/instance_test.go | 208 ++++++++++++++- .../tsdb/bksql/querystring_test.go | 105 -------- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 240 ++++++++++++++++++ .../tsdb/bksql/sqlExpr/base_test.go | 139 ++++++++++ .../{querystring.go => sqlExpr/doris.go} | 74 +++--- .../tsdb/bksql/sqlExpr/doris_test.go | 240 ++++++++++++++++++ 11 files changed, 977 insertions(+), 201 deletions(-) delete mode 100644 pkg/unify-query/tsdb/bksql/querystring_test.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/base.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go rename pkg/unify-query/tsdb/bksql/{querystring.go => sqlExpr/doris.go} (75%) create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go diff --git a/.gitignore b/.gitignore index 955d14f6c..f065a013c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ go.work.sum .vscode /.gtm.yaml .python-version + +bkcodeai.json diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 45570fe2c..1081d8eec 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -29,6 +29,24 @@ const ( UUID = "query_uuid" ) +const ( + ConditionEqual = "eq" + ConditionNotEqual = "ne" + ConditionRegEqual = "req" + ConditionNotRegEqual = "nreq" + ConditionContains = "contains" + ConditionNotContains = "ncontains" + + ConditionExisted = "existed" + ConditionNotExisted = "nexisted" + + ConditionExact = "exact" + ConditionGt = "gt" + ConditionGte = "gte" + ConditionLt = "lt" + ConditionLte = "lte" +) + type VmCondition string type TimeField struct { @@ -40,7 +58,8 @@ type TimeField struct { // Aggregate 聚合方法 type Aggregate struct { - Name string + Name string + Dimensions []string Without bool diff --git a/pkg/unify-query/tsdb/bksql/client_test.go b/pkg/unify-query/tsdb/bksql/client_test.go index 8761c7358..593bb0a52 100644 --- a/pkg/unify-query/tsdb/bksql/client_test.go +++ b/pkg/unify-query/tsdb/bksql/client_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -21,15 +21,16 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) var ( - client *Client + client *bksql.Client ) -func MockClient() *Client { +func MockClient() *bksql.Client { if client == nil { - client = (&Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) + client = (&bksql.Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) } return client @@ -43,10 +44,10 @@ func TestClient_QuerySync(t *testing.T) { end := time.UnixMilli(1729838923416) mock.BkSQL.Set(map[string]any{ - `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &Result{ + `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &bksql.Result{ Result: true, - Code: StatusOK, - Data: &QuerySyncResultData{ + Code: bksql.StatusOK, + Data: &bksql.QuerySyncResultData{ TotalRecords: 5, SelectFieldsOrder: []string{ "dtEventTimeStamp", @@ -94,8 +95,8 @@ func TestClient_QuerySync(t *testing.T) { nil, ) - assert.Equal(t, StatusOK, res.Code) - d, ok := res.Data.(*QuerySyncResultData) + assert.Equal(t, bksql.StatusOK, res.Code) + d, ok := res.Data.(*bksql.QuerySyncResultData) assert.True(t, ok) assert.Equal(t, d.TotalRecords, 5) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 5edbbbb9f..9cb910057 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -21,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) const ( @@ -132,7 +133,7 @@ func (f *QueryFactory) ParserQuery() (err error) { return } -func (f *QueryFactory) getTheDateFilters() (theDateFilter string, err error) { +func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { @@ -157,6 +158,48 @@ func (f *QueryFactory) getTheDateFilters() (theDateFilter string, err error) { return } +func (f *QueryFactory) BuildWhere() (string, error) { + var s []string + s = append(s, fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) + + theDateFilter, err := f.getTheDateIndexFilters() + if err != nil { + return "", err + } + if theDateFilter != "" { + s = append(s, theDateFilter) + } + + fieldsMap := make(map[string]string) + expr := sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap) + + // QueryString to sql + if f.query.QueryString != "" { + qs, err := expr.ParserQueryString(f.query.QueryString) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + // AllConditions to sql + if len(f.query.AllConditions) > 0 { + qs, err := expr.ParserAllConditions(f.query.AllConditions) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + return strings.Join(s, " AND "), nil +} + func (f *QueryFactory) SQL() (sql string, err error) { f.sql.Reset() err = f.ParserQuery() @@ -175,35 +218,14 @@ func (f *QueryFactory) SQL() (sql string, err error) { db += "." + f.query.Measurement } f.write(db) - f.write("WHERE") - f.write(fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) - theDateFilter, err := f.getTheDateFilters() + where, err := f.BuildWhere() if err != nil { return } - if theDateFilter != "" { - f.write("AND") - f.write(theDateFilter) - } - - if f.query.BkSqlCondition != "" { - f.write("AND") - f.write("(" + f.query.BkSqlCondition + ")") - } - - // 拼接 querystring 数据 - if f.query.QueryString != "" { - // 字段类型 - fieldsMap := make(map[string]string) - qsSql, qsErr := NewDorisSQLExpr(f.query.QueryString).WithFieldsMap(fieldsMap).Parser() - if qsErr != nil { - err = qsErr - return - } - - f.write("AND") - f.write(qsSql) + if where != "" { + f.write("WHERE") + f.write(where) } if len(f.groups) > 0 { diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index b2f0ba189..7f0a98383 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -18,6 +18,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) func TestNewSqlFactory(t *testing.T) { @@ -45,12 +46,20 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 0, - Orders: metadata.Orders{"ip": true}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + }, + }, + From: 0, + Size: 0, + Orders: metadata.Orders{"ip": true}, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", }, "sum-with-promql-1": { query: &metadata.Query{ @@ -65,12 +74,20 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 10, - Orders: nil, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + }, + }, + From: 0, + Size: 10, + Orders: nil, }, - expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip` LIMIT 10", + expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, "count-with-count-promql-1": { query: &metadata.Query{ @@ -86,9 +103,8 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, "count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -110,9 +126,17 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "gseIndex", + Value: []string{"0"}, + Operator: metadata.ConditionGt, + }, + }, + }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { @@ -125,7 +149,7 @@ func TestNewSqlFactory(t *testing.T) { } log.Infof(ctx, "start: %s, end: %s", c.start, c.end) - fact := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) sql, err := fact.SQL() assert.Nil(t, err) assert.Equal(t, c.expected, sql) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 568125764..e169bdcfc 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -21,6 +21,8 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) func TestInstance_QueryRaw(t *testing.T) { @@ -28,7 +30,7 @@ func TestInstance_QueryRaw(t *testing.T) { ctx := context.Background() mock.Init() - ins, err := NewInstance(ctx, &Options{ + ins, err := bksql.NewInstance(ctx, &bksql.Options{ Address: mock.BkSQLUrl, Timeout: time.Minute, MaxLimit: 1e4, @@ -41,7 +43,7 @@ func TestInstance_QueryRaw(t *testing.T) { } mock.BkSQL.Set(map[string]any{ - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND (namespace = 'gz100' OR namespace = 'bgp2\\-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND ((namespace = 'gz100') OR (namespace = 'bgp2-new')) LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) @@ -64,8 +66,17 @@ func TestInstance_QueryRaw(t *testing.T) { TableID: tableID, DB: db, MetricName: field, - BkSqlCondition: "namespace = 'gz100' OR namespace = 'bgp2\\-new'", + BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", OffsetInfo: metadata.OffSetInfo{Limit: 10}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "namespace", + Operator: metadata.ConditionEqual, + Value: []string{"gz100", "bgp2-new"}, + }, + }, + }, }, expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":269,"timestamp":1730118660000},{"value":271,"timestamp":1730118480000},{"value":267,"timestamp":1730118540000},{"value":274,"timestamp":1730118600000},{"value":279,"timestamp":1730118420000}],"exemplars":null,"histograms":null}]`, }, @@ -136,7 +147,7 @@ func TestInstance_bkSql(t *testing.T) { query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", - BkSqlCondition: "namespace REGEXP '^(bgp2\\-new|gz100)$'", + BkSqlCondition: "`namespace` IN ('bgp2-new', 'gz100')", Aggregates: metadata.Aggregates{ { Name: "count", @@ -144,8 +155,17 @@ func TestInstance_bkSql(t *testing.T) { Window: time.Second * 15, }, }, + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"bgp2-new", "gz100"}, + }, + }, + }, }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (namespace REGEXP '^(bgp2\\-new|gz100)$') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", }, { query: &metadata.Query{ @@ -241,7 +261,14 @@ func TestInstance_bkSql(t *testing.T) { c.end = end } - sql, err := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end).SQL() + condition, err := sqlExpr.GetSQLExpr(c.query.Measurement).WithFieldsMap(nil).ParserAllConditions(c.query.AllConditions) + assert.Nil(t, err) + if err == nil { + assert.Equal(t, c.query.BkSqlCondition, condition) + } + + fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + sql, err := fact.SQL() assert.Nil(t, err) if err == nil { assert.Equal(t, c.expected, sql) @@ -249,3 +276,170 @@ func TestInstance_bkSql(t *testing.T) { }) } } + +func TestInstance_bkSql_EdgeCases(t *testing.T) { + mock.Init() + + // 基础时间范围 + baseStart := time.UnixMilli(1718189940000) + baseEnd := time.UnixMilli(1718193555000) + + // 跨天时间范围 + crossDayStart := time.Unix(1733756400, 0) // 2024-12-09 00:00:00 + crossDayEnd := time.Unix(1733846399, 0) // 2024-12-09 23:59:59 + + testCases := []struct { + name string + start time.Time + end time.Time + query *metadata.Query + expected string + }{ + // 测试用例1: 无聚合函数的原始查询 + { + name: "raw query without aggregation", + query: &metadata.Query{ + DB: "test_db", + Field: "value", + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `test_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例2: 多聚合函数组合 + { + name: "multiple aggregates", + query: &metadata.Query{ + DB: "metrics_db", + Field: "temperature", + Aggregates: metadata.Aggregates{ + {Name: "max"}, + {Name: "min"}, + }, + }, + expected: "SELECT MAX(`temperature`) AS `_value_`, MIN(`temperature`) AS `_value_` FROM `metrics_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例3: 复杂条件组合 + { + name: "complex conditions", + query: &metadata.Query{ + DB: "security_logs", + Field: "duration", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "severity", + Operator: metadata.ConditionEqual, + Value: []string{"high", "critical"}, + }, + { + DimensionName: "source_ip", + Operator: metadata.ConditionNotContains, + Value: []string{"192.168.1.1"}, + }, + }, + }, + }, + expected: "SELECT *, `duration` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `security_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `severity` IN ('high', 'critical') AND `source_ip` != '192.168.1.1'", + }, + + // 测试用例4: 多字段排序 + { + name: "multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "amount", + Orders: map[string]bool{ + "timestamp": true, + "account_id": false, + }, + }, + expected: "SELECT *, `amount` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `account_id` DESC, `timestamp` ASC", + }, + + // 测试用例5: 特殊字符转义 + { + name: "special characters in fields", + query: &metadata.Query{ + DB: "special_metrics", + Measurement: "select", // 保留字作为measurement + Field: "*", + Aggregates: metadata.Aggregates{ + {Name: "sum"}, + }, + }, + expected: "SELECT SUM(`*`) AS `_value_` FROM `special_metrics`.select WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例6: 零窗口时间 + { + name: "zero window size", + query: &metadata.Query{ + DB: "time_series_data", + Field: "value", + Aggregates: metadata.Aggregates{ + { + Name: "avg", + Window: 0, + }, + }, + }, + expected: "SELECT AVG(`value`) AS `_value_` FROM `time_series_data` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例7: 跨多天的时间范围 + { + name: "multi-day time range", + start: crossDayStart, + end: crossDayEnd, + query: &metadata.Query{ + DB: "daily_metrics", + Field: "active_users", + Aggregates: metadata.Aggregates{ + {Name: "count"}, + }, + }, + expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + + // 设置默认时间范围 + start := tc.start + if start.IsZero() { + start = baseStart + } + end := tc.end + if end.IsZero() { + end = baseEnd + } + + // 条件解析验证 + if len(tc.query.AllConditions) > 0 { + condition, err := sqlExpr.GetSQLExpr(tc.query.Measurement).WithFieldsMap(nil).ParserAllConditions(tc.query.AllConditions) + assert.Nil(t, err) + if err == nil { + assert.NotEmpty(t, condition, "Parsed condition should not be empty") + } + } + + // SQL生成验证 + fact := bksql.NewQueryFactory(ctx, tc.query).WithRangeTime(start, end) + generatedSQL, err := fact.SQL() + + assert.Nil(t, err) + if err == nil { + assert.Equal(t, tc.expected, generatedSQL) + } + + // 验证时间条件 + if tc.start.IsZero() && tc.end.IsZero() { + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` < %d", baseEnd.UnixMilli())) + } + }) + } +} diff --git a/pkg/unify-query/tsdb/bksql/querystring_test.go b/pkg/unify-query/tsdb/bksql/querystring_test.go deleted file mode 100644 index be146637b..000000000 --- a/pkg/unify-query/tsdb/bksql/querystring_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Tencent is pleased to support the open source community by making -// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. -// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. -// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://opensource.org/licenses/MIT -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -package bksql - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestQueryStringToSQL(t *testing.T) { - tests := []struct { - name string - input string - want string - err string - }{ - { - name: "simple match", - input: "name:test", - want: "`name` = 'test'", - }, - { - name: "one word", - input: "test", - err: "doris 不支持全字段检索: test", - }, - { - name: "complex nested query", - input: "(a:1 AND (b:2 OR c:3)) OR NOT d:4", - want: "( ( `a` = '1' AND ( `b` = '2' OR `c` = '3' ) ) OR NOT ( `d` = '4' ) )", - }, - { - name: "invalid syntax", - input: "name:test AND OR", - err: "syntax error: unexpected tOR", - }, - { - name: "empty input", - input: "", - }, - { - name: "OR expression with multiple terms", - input: "a:1 OR b:2 OR c:3", - want: "( `a` = '1' OR ( `b` = '2' OR `c` = '3' ) )", - }, - { - name: "mixed AND/OR with proper precedence", - input: "a:1 AND b:2 OR c:3", - want: "( `a` = '1' AND ( `b` = '2' OR `c` = '3' ) )", - }, - { - name: "exact match with quotes", - input: "name:\"exact match\"", - want: "`name` = 'exact match'", - }, - { - name: "numeric equality", - input: "age:25", - want: "`age` = '25'", - }, - { - name: "date range query", - input: "timestamp:[2023-01-01 TO 2023-12-31]", - want: "( timestamp >= '2023-01-01' AND timestamp <= '2023-12-31' )", - err: "syntax error: unexpected tSTRING, expecting tPHRASE or tNUMBER or tSTAR or tMINUS", - }, - { - name: "invalid field name", - input: "123field:value", - want: "`123field` = 'value'", - }, - { - name: "text filter", - input: "text:value", - want: "`text` LIKE '%value%'", - }, - { - name: "object field", - input: "__ext.container_name: value", - want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := NewDorisSQLExpr(tt.input).WithFieldsMap(map[string]string{ - "text": Text, - }).Parser() - if err != nil { - assert.Equal(t, err.Error(), tt.err) - return - } - - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go new file mode 100644 index 000000000..fc8a9bdc7 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -0,0 +1,240 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr + +import ( + "fmt" + "strings" + "sync" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" +) + +// ErrorMatchAll 定义全字段检索错误提示信息 +var ( + ErrorMatchAll = "不支持全字段检索" +) + +// SQLExpr 定义SQL表达式生成接口 +// 接口包含字段映射、查询字符串解析、全条件解析等功能 +type SQLExpr interface { + // WithFieldsMap 设置字段类型 + WithFieldsMap(fieldsMap map[string]string) SQLExpr + // WithTransformDimension 设置字段转换方法 + WithTransformDimension(func(string) string) SQLExpr + // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 + ParserQueryString(qs string) (string, error) + // ParserAllConditions 解析全量条件生成SQL条件表达式 + ParserAllConditions(allConditions metadata.AllConditions) (string, error) +} + +// SQL表达式注册管理相关变量 +var ( + _ SQLExpr = (*DefaultSQLExpr)(nil) // 接口实现检查 + + lock sync.RWMutex // 读写锁用于并发安全 + ExprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 +) + +// GetSQLExpr 获取指定key的SQL表达式实现 +// 参数： +// +// key - 注册时使用的标识符 +// +// 返回值： +// +// 找到返回对应实现，未找到返回默认实现 +func GetSQLExpr(key string) SQLExpr { + lock.RLock() + defer lock.RUnlock() + if sqlExpr, ok := ExprMap[key]; ok { + return sqlExpr + } else { + return &DefaultSQLExpr{} + } +} + +// Register 注册SQL表达式实现 +// 参数： +// +// key - 实现标识符 +// sqlExpr - 具体的SQL表达式实现 +func Register(key string, sqlExpr SQLExpr) { + lock.Lock() + defer lock.Unlock() + ExprMap[key] = sqlExpr +} + +// UnRegister 注销指定key的SQL表达式实现 +// 参数： +// +// key - 要注销的实现标识符 +func UnRegister(key string) { + lock.Lock() + defer lock.Unlock() + if _, ok := ExprMap[key]; ok { + delete(ExprMap, key) + } +} + +// DefaultSQLExpr SQL表达式默认实现 +type DefaultSQLExpr struct { + dimTransform func(string) string + fieldsMap map[string]string +} + +func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { + d.fieldsMap = fieldsMap + return d +} + +// WithTransformDimension 实现设置字段转换方法 +func (d *DefaultSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr { + d.dimTransform = fn + return d +} + +// ParserQueryString 解析查询字符串（当前实现返回空） +func (d *DefaultSQLExpr) ParserQueryString(qs string) (string, error) { + return "", nil +} + +// ParserAllConditions 解析全量条件生成SQL条件表达式 +// 实现逻辑： +// 1. 将多个AND条件组合成OR条件 +// 2. 当有多个OR条件时用括号包裹 +func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + var ( + orConditions []string + ) + + // 遍历所有OR条件组 + for _, conditions := range allConditions { + var andConditions []string + // 处理每个AND条件组 + for _, cond := range conditions { + buildCondition, err := d.buildCondition(cond) + if err != nil { + return "", err + } + if buildCondition != "" { + andConditions = append(andConditions, buildCondition) + } + } + // 合并AND条件 + if len(andConditions) > 0 { + orConditions = append(orConditions, strings.Join(andConditions, " AND ")) + } + } + + // 处理最终OR条件组合 + if len(orConditions) > 0 { + if len(orConditions) == 1 { + return orConditions[0], nil + } + return fmt.Sprintf("(%s)", strings.Join(orConditions, " OR ")), nil + } + + return "", nil +} + +// buildDimensionName 构建维度字段名称（添加反引号包裹） +func (d *DefaultSQLExpr) buildDimensionName(dim string) string { + return fmt.Sprintf("`%s`", dim) +} + +// buildCondition 构建单个条件表达式 +// 参数： +// +// c - 条件字段对象 +// +// 返回值： +// +// 生成的SQL条件表达式字符串和错误信息 +func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { + if len(c.Value) == 0 { + return "", nil + } + + var ( + key string + op string + val string + ) + + if d.dimTransform != nil { + key = d.dimTransform(c.DimensionName) + } else { + key = d.buildDimensionName(c.DimensionName) + } + + // 根据操作符类型生成不同的SQL表达式 + switch c.Operator { + // 处理等于类操作符（=, IN） + case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) == 1 { + if c.IsWildcard { + op = "LIKE" + val = fmt.Sprintf("'%%%s%%'", c.Value[0]) + } else { + op = "=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } + } else { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理不等于类操作符（!=, NOT IN） + case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) == 1 { + op = "!=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } else { + op = "NOT IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理正则表达式匹配 + case metadata.ConditionRegEqual: + op = "REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) // 多个值用|连接 + case metadata.ConditionNotRegEqual: + op = "NOT REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) + // 处理数值比较操作符（>, >=, <, <=） + case metadata.ConditionGt: + op = ">" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionGte: + op = ">=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLt: + op = "<" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLte: + op = "<=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + default: + return "", fmt.Errorf("unknown operator %s", c.Operator) + } + + return fmt.Sprintf("%s %s %s", key, op, val), nil +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go new file mode 100644 index 000000000..46f46b56b --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go @@ -0,0 +1,139 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +// TestParserAllConditions 测试全条件解析的主测试函数 +func TestParserAllConditions(t *testing.T) { + d := &sqlExpr.DefaultSQLExpr{} + + t.Run("空条件测试", func(t *testing.T) { + conditions := metadata.AllConditions{} + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "", result) + }) + + t.Run("单OR组单AND条件", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "host", Operator: metadata.ConditionEqual, Value: []string{"server1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`host` = 'server1'", result) + }) + + t.Run("多OR组组合", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "os", Operator: metadata.ConditionEqual, Value: []string{"linux"}}, + {DimensionName: "status", Operator: metadata.ConditionGt, Value: []string{"5"}}, + }, + { + {DimensionName: "region", Operator: metadata.ConditionEqual, Value: []string{"north", "south"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`os` = 'linux' AND `status` > 5 OR `region` IN ('north', 'south'))" + assert.Equal(t, expected, result) + }) + + t.Run("包含空AND条件过滤", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "log", Operator: metadata.ConditionRegEqual, Value: []string{}}, // 空值条件 + {DimensionName: "level", Operator: metadata.ConditionNotEqual, Value: []string{"debug"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`level` != 'debug'", result) + }) + + t.Run("混合操作符测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "cpu", Operator: metadata.ConditionGte, Value: []string{"80"}}, + {DimensionName: "memory", Operator: metadata.ConditionLt, Value: []string{"90"}}, + }, + { + {DimensionName: "service", Operator: metadata.ConditionContains, Value: []string{"api", "db"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`cpu` >= 80 AND `memory` < 90 OR `service` IN ('api', 'db'))" + assert.Equal(t, expected, result) + }) + + t.Run("错误条件传递测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "error", Operator: metadata.ConditionGt, Value: []string{"1", "2"}}, + }, + } + _, err := d.ParserAllConditions(conditions) + assert.Error(t, err) + assert.Contains(t, err.Error(), "operator > only support 1 value") + }) + + t.Run("多层嵌套组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "A", Operator: metadata.ConditionEqual, Value: []string{"1"}}, + {DimensionName: "B", Operator: metadata.ConditionNotEqual, Value: []string{"2"}}, + }, + { + {DimensionName: "C", Operator: metadata.ConditionRegEqual, Value: []string{"test.*"}}, + }, + { + {DimensionName: "D", Operator: metadata.ConditionLte, Value: []string{"100"}}, + {DimensionName: "E", Operator: metadata.ConditionContains, Value: []string{"x", "y"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`A` = '1' AND `B` != '2' OR `C` REGEXP 'test.*' OR `D` <= 100 AND `E` IN ('x', 'y'))" + assert.Equal(t, expected, result) + }) + + t.Run("单值IN转换测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "version", Operator: metadata.ConditionEqual, Value: []string{"v1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`version` = 'v1'", result) + }) + + t.Run("正则表达式组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "path", Operator: metadata.ConditionRegEqual, Value: []string{"^/api", "v2$"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`path` REGEXP '^/api|v2$'", result) + }) +} diff --git a/pkg/unify-query/tsdb/bksql/querystring.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go similarity index 75% rename from pkg/unify-query/tsdb/bksql/querystring.go rename to pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index e3ea175ae..42ffbfb0f 100644 --- a/pkg/unify-query/tsdb/bksql/querystring.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -7,56 +7,37 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package sqlExpr import ( "fmt" "strings" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" ) const ( - KeyWord = "keyword" - Text = "text" - Integer = "integer" - Long = "long" - Date = "date" -) - -var ( - ErrorMatchAll = "doris 不支持全字段检索" + Doris = "doris" + DorisTypeText = "text" ) type DorisSQLExpr struct { - qs string + DefaultSQLExpr + fieldsMap map[string]string } -func NewDorisSQLExpr(qs string) *DorisSQLExpr { - return &DorisSQLExpr{ - qs: qs, - } -} +var _ SQLExpr = (*DorisSQLExpr)(nil) -func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) *DorisSQLExpr { +func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { s.fieldsMap = fieldsMap + s.DefaultSQLExpr.WithFieldsMap(fieldsMap).WithTransformDimension(s.fieldFormat) return s } -func (s *DorisSQLExpr) checkMatchALL(k string) bool { - if s.fieldsMap != nil { - if t, ok := s.fieldsMap[k]; ok { - if t == Text { - return true - } - } - } - return false -} - -func (s *DorisSQLExpr) Parser() (string, error) { - expr, err := querystring.Parse(s.qs) +func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { + expr, err := querystring.Parse(qs) if err != nil { return "", err } @@ -67,6 +48,21 @@ func (s *DorisSQLExpr) Parser() (string, error) { return s.walk(expr) } +func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + return s.DefaultSQLExpr.ParserAllConditions(allConditions) +} + +func (s *DorisSQLExpr) checkMatchALL(k string) bool { + if s.fieldsMap != nil { + if t, ok := s.fieldsMap[k]; ok { + if t == DorisTypeText { + return true + } + } + } + return false +} + func (s *DorisSQLExpr) fieldFormat(field string) string { fs := strings.Split(field, ".") if len(fs) > 1 { @@ -92,7 +88,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { if err != nil { return "", err } - return fmt.Sprintf("NOT ( %s )", left), nil + return fmt.Sprintf("NOT (%s)", left), nil case *querystring.OrExpr: left, err = s.walk(c.Left) if err != nil { @@ -102,7 +98,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { if err != nil { return "", err } - return fmt.Sprintf("( %s OR %s )", left, right), nil + return fmt.Sprintf("(%s OR %s)", left, right), nil case *querystring.AndExpr: left, err = s.walk(c.Left) if err != nil { @@ -112,17 +108,17 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { if err != nil { return "", err } - return fmt.Sprintf("( %s AND %s )", left, right), nil + return fmt.Sprintf("%s AND %s", left, right), nil case *querystring.WildcardExpr: if c.Field == "" { - err = fmt.Errorf(ErrorMatchAll) + err = fmt.Errorf(Doris + " " + ErrorMatchAll) return "", err } return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil case *querystring.MatchExpr: if c.Field == "" { - err = fmt.Errorf(ErrorMatchAll + ": " + c.Value) + err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) return "", err } @@ -133,7 +129,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return fmt.Sprintf("%s = '%s'", s.fieldFormat(c.Field), c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { - err = fmt.Errorf(ErrorMatchAll) + err = fmt.Errorf(Doris + " " + ErrorMatchAll) return "", err } @@ -161,10 +157,14 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { end = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.End) } - return fmt.Sprintf("( %s )", strings.Join([]string{start, end}, " AND ")), nil + return fmt.Sprintf("%s", strings.Join([]string{start, end}, " AND ")), nil default: err = fmt.Errorf("expr type is not match %T", e) } return "", err } + +func init() { + Register(Doris, &DorisSQLExpr{}) +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go new file mode 100644 index 000000000..a00939783 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -0,0 +1,240 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +func TestDorisSQLExpr_ParserQueryString(t *testing.T) { + tests := []struct { + name string + input string + want string + err string + }{ + { + name: "simple match", + input: "name:test", + want: "`name` = 'test'", + }, + { + name: "one word", + input: "test", + err: "doris 不支持全字段检索: test", + }, + { + name: "complex nested query", + input: "(a:1 AND (b:2 OR c:3)) OR NOT d:4", + want: "(`a` = '1' AND (`b` = '2' OR `c` = '3') OR NOT (`d` = '4'))", + }, + { + name: "invalid syntax", + input: "name:test AND OR", + err: "syntax error: unexpected tOR", + }, + { + name: "empty input", + input: "", + }, + { + name: "OR expression with multiple terms", + input: "a:1 OR b:2 OR c:3", + want: "(`a` = '1' OR (`b` = '2' OR `c` = '3'))", + }, + { + name: "mixed AND/OR with proper precedence", + input: "a:1 AND b:2 OR c:3", + want: "`a` = '1' AND (`b` = '2' OR `c` = '3')", + }, + { + name: "exact match with quotes", + input: "name:\"exact match\"", + want: "`name` = 'exact match'", + }, + { + name: "numeric equality", + input: "age:25", + want: "`age` = '25'", + }, + { + name: "date range query", + input: "timestamp:[2023-01-01 TO 2023-12-31]", + want: "( timestamp >= '2023-01-01' AND timestamp <= '2023-12-31' )", + err: "syntax error: unexpected tSTRING, expecting tPHRASE or tNUMBER or tSTAR or tMINUS", + }, + { + name: "invalid field name", + input: "123field:value", + want: "`123field` = 'value'", + }, + { + name: "text filter", + input: "text:value", + want: "`text` LIKE '%value%'", + }, + { + name: "object field", + input: "__ext.container_name: value", + want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "text": sqlExpr.DorisTypeText, + }).ParserQueryString(tt.input) + if err != nil { + assert.Equal(t, err.Error(), tt.err) + return + } + + assert.Equal(t, tt.want, got) + }) + } +} + +// TestDorisSQLExpr_ParserAllConditions 单元测试 +func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { + tests := []struct { + name string + condition metadata.AllConditions + want string + wantErr error + }{ + { + name: "test object field condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"test-value"}, + Operator: metadata.ConditionEqual, + }, + { + DimensionName: "tag", + Value: []string{"test"}, + Operator: metadata.ConditionNotEqual, + }, + }, + }, + want: "CAST(object[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", + }, + { + name: "test text field wildcard", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"partial"}, + Operator: metadata.ConditionContains, + IsWildcard: true, + }, + }, + }, + want: `CAST(object["field"] AS STRING) LIKE '%partial%'`, + }, + { + name: "test OR condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "status", + Value: []string{"running"}, + Operator: metadata.ConditionEqual, + }, + }, + { + { + DimensionName: "code", + Value: []string{"500"}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "(`status` = 'running' OR `code` = '500')", + }, + { + name: "test numeric field without cast", + condition: metadata.AllConditions{ + { + { + DimensionName: "cpu_usage", + Value: []string{"80"}, + Operator: metadata.ConditionGt, + }, + }, + }, + want: "`cpu_usage` > 80", + }, + { + name: "test IN operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "env", + Value: []string{"prod", "test"}, + Operator: metadata.ConditionContains, + }, + }, + }, + want: "`env` IN ('prod', 'test')", + }, + { + name: "test empty value", + condition: metadata.AllConditions{ + { + { + DimensionName: "host", + Value: []string{}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "", + }, + { + name: "test invalid operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "time", + Value: []string{"2023"}, + Operator: "unknown", + }, + }, + }, + wantErr: fmt.Errorf("unknown operator unknown"), + }, + } + + e := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "object.field": sqlExpr.DorisTypeText, + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := e.ParserAllConditions(tt.condition) + + if err != nil { + assert.Equal(t, tt.wantErr, err) + return + } + assert.Equal(t, tt.want, got) + }) + } +} From c74f7e54e744a4dcbce82205e253ba1a8ebc063c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 11:48:06 +0800 Subject: [PATCH 019/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index a00939783..636737517 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -188,7 +188,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { { DimensionName: "env", Value: []string{"prod", "test"}, - Operator: metadata.ConditionContains, + Operator: metadata.ConditionEqual, }, }, }, From b5c8b06b44ba0c9ef4e6cf4435ccda91663ec97c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 11:49:48 +0800 Subject: [PATCH 020/104] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E5=8D=95?= =?UTF-8?q?=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 636737517..a00939783 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -188,7 +188,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { { DimensionName: "env", Value: []string{"prod", "test"}, - Operator: metadata.ConditionEqual, + Operator: metadata.ConditionContains, }, }, }, From 8970c5d7d81ae65b7aed40c38b5aa1100c480ef2 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 11:54:41 +0800 Subject: [PATCH 021/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504=20#=20R?= =?UTF-8?q?eviewed,=20transaction=20id:=2031530?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + pkg/collector/VERSION | 2 +- pkg/collector/example/example.yml | 8 +- pkg/collector/example/platform.yml | 7 - .../processor/resourcefilter/config.go | 7 +- pkg/collector/processor/resourcefilter/doc.go | 2 - .../processor/resourcefilter/factory.go | 28 +- .../processor/resourcefilter/factory_test.go | 30 ++- .../{dimscache => k8scache}/cache.go | 65 +++-- .../{dimscache => k8scache}/cache_test.go | 47 ++-- .../x86_64/etc/bk-collector-platform.conf.tpl | 5 - .../operator/objectsref/controller.go | 80 ++---- pkg/operator/operator/objectsref/lookup.go | 10 +- .../operator/objectsref/lookup_test.go | 12 +- pkg/operator/operator/objectsref/pod.go | 193 ++++++++++++++ .../{controller_test.go => pod_test.go} | 8 +- pkg/operator/operator/objectsref/relabels.go | 4 +- .../operator/objectsref/relabels_test.go | 3 +- pkg/operator/operator/objectsref/relation.go | 18 +- .../operator/objectsref/relation_test.go | 14 +- pkg/operator/operator/objectsref/ring/ring.go | 140 ++++++++++ .../operator/objectsref/ring/ring_test.go | 77 ++++++ pkg/operator/operator/server.go | 38 ++- pkg/unify-query/internal/querystring/expr.go | 5 + pkg/unify-query/metadata/struct.go | 21 +- pkg/unify-query/query/structured/query_ts.go | 32 ++- pkg/unify-query/service/http/query.go | 91 ++++--- pkg/unify-query/service/influxdb/hook.go | 3 + pkg/unify-query/service/influxdb/service.go | 5 +- pkg/unify-query/service/influxdb/settings.go | 4 + pkg/unify-query/tsdb/bksql/client_test.go | 19 +- pkg/unify-query/tsdb/bksql/format.go | 59 ++++- pkg/unify-query/tsdb/bksql/format_test.go | 56 ++-- pkg/unify-query/tsdb/bksql/instance.go | 101 ++++++-- pkg/unify-query/tsdb/bksql/instance_test.go | 208 ++++++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 240 ++++++++++++++++++ .../tsdb/bksql/sqlExpr/base_test.go | 139 ++++++++++ pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 170 +++++++++++++ .../tsdb/bksql/sqlExpr/doris_test.go | 240 ++++++++++++++++++ .../tsdb/elasticsearch/instance.go | 17 +- 40 files changed, 1893 insertions(+), 317 deletions(-) rename pkg/collector/processor/resourcefilter/{dimscache => k8scache}/cache.go (73%) rename pkg/collector/processor/resourcefilter/{dimscache => k8scache}/cache_test.go (63%) create mode 100644 pkg/operator/operator/objectsref/pod.go rename pkg/operator/operator/objectsref/{controller_test.go => pod_test.go} (93%) create mode 100644 pkg/operator/operator/objectsref/ring/ring.go create mode 100644 pkg/operator/operator/objectsref/ring/ring_test.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/base.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/doris.go create mode 100644 pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go diff --git a/.gitignore b/.gitignore index 955d14f6c..f065a013c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ go.work.sum .vscode /.gtm.yaml .python-version + +bkcodeai.json diff --git a/pkg/collector/VERSION b/pkg/collector/VERSION index feb701544..e8a139fdb 100644 --- a/pkg/collector/VERSION +++ b/pkg/collector/VERSION @@ -1 +1 @@ -0.76.x \ No newline at end of file +0.77.x \ No newline at end of file diff --git a/pkg/collector/example/example.yml b/pkg/collector/example/example.yml index d146c15d5..7ac39c7d3 100644 --- a/pkg/collector/example/example.yml +++ b/pkg/collector/example/example.yml @@ -399,13 +399,7 @@ bk-collector: drop: keys: - "resource.bk.data.token" -# from_cache: -# key: "resource.service.name" -# cache: -# key: "service.name" -# url: "http://localhost:8990/service_info" -# interval: "1m" -# timeout: "1m" + from_record: - source: "request.client.ip" destination: "resource.client.ip" diff --git a/pkg/collector/example/platform.yml b/pkg/collector/example/platform.yml index 109fb4991..38819dd98 100644 --- a/pkg/collector/example/platform.yml +++ b/pkg/collector/example/platform.yml @@ -25,10 +25,3 @@ processor: drop: keys: - "resource.bk.data.token" -# from_cache: -# key: "resource.service.name" -# cache: -# key: "service.name" -# url: "http://localhost:8990/service_info" -# interval: "1m" -# timeout: "1m" diff --git a/pkg/collector/processor/resourcefilter/config.go b/pkg/collector/processor/resourcefilter/config.go index 94b0f0351..0b578a897 100644 --- a/pkg/collector/processor/resourcefilter/config.go +++ b/pkg/collector/processor/resourcefilter/config.go @@ -12,7 +12,7 @@ package resourcefilter import ( "strings" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor/resourcefilter/dimscache" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor/resourcefilter/k8scache" ) type Config struct { @@ -61,9 +61,8 @@ type AssembleAction struct { } type FromCacheAction struct { - Key string `config:"key" mapstructure:"key"` - Dimensions []string `config:"dimensions" mapstructure:"dimensions"` - Cache dimscache.Config `config:"cache" mapstructure:"cache"` + Key string `config:"key" mapstructure:"key"` + Cache k8scache.Config `config:"cache" mapstructure:"cache"` keys []string } diff --git a/pkg/collector/processor/resourcefilter/doc.go b/pkg/collector/processor/resourcefilter/doc.go index 8a6a7fc29..498764798 100644 --- a/pkg/collector/processor/resourcefilter/doc.go +++ b/pkg/collector/processor/resourcefilter/doc.go @@ -53,9 +53,7 @@ processor: config: from_cache: key: "resource.net.host.ip|resource.client.ip" - dimensions: ["k8s.namespace.name","k8s.pod.name","k8s.pod.ip","k8s.bcs.cluster.id"] cache: - key: "k8s.pod.ip" url: http://localhost:8080/pods interval: "1m" timeout: "1m" diff --git a/pkg/collector/processor/resourcefilter/factory.go b/pkg/collector/processor/resourcefilter/factory.go index 6b593c7bb..53774a6b3 100644 --- a/pkg/collector/processor/resourcefilter/factory.go +++ b/pkg/collector/processor/resourcefilter/factory.go @@ -21,7 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/define" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/mapstructure" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor/resourcefilter/dimscache" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor/resourcefilter/k8scache" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" ) @@ -44,7 +44,7 @@ func newFactory(conf map[string]interface{}, customized []processor.SubConfigPro c.Clean() configs.SetGlobal(*c) - cache := dimscache.New(&c.FromCache.Cache) + cache := k8scache.New(&c.FromCache.Cache) cache.Sync() caches.SetGlobal(cache) @@ -57,7 +57,7 @@ func newFactory(conf map[string]interface{}, customized []processor.SubConfigPro cfg.Clean() configs.Set(custom.Token, custom.Type, custom.ID, *cfg) - customCache := dimscache.New(&cfg.FromCache.Cache) + customCache := k8scache.New(&cfg.FromCache.Cache) customCache.Sync() caches.Set(custom.Token, custom.Type, custom.ID, customCache) } @@ -72,7 +72,7 @@ func newFactory(conf map[string]interface{}, customized []processor.SubConfigPro type resourceFilter struct { processor.CommonProcessor configs *confengine.TierConfig // type: Config - caches *confengine.TierConfig // type dimscache.Cache + caches *confengine.TierConfig // type k8scache.Cache } func (p *resourceFilter) Name() string { @@ -96,25 +96,25 @@ func (p *resourceFilter) Reload(config map[string]interface{}, customized []proc equal := processor.DiffMainConfig(p.MainConfig(), config) if equal { - f.caches.GetGlobal().(dimscache.Cache).Clean() + f.caches.GetGlobal().(k8scache.Cache).Clean() } else { - p.caches.GetGlobal().(dimscache.Cache).Clean() + p.caches.GetGlobal().(k8scache.Cache).Clean() p.caches.SetGlobal(f.caches.GetGlobal()) } diffRet := processor.DiffCustomizedConfig(p.SubConfigs(), customized) for _, obj := range diffRet.Keep { - f.caches.Get(obj.Token, obj.Type, obj.ID).(dimscache.Cache).Clean() + f.caches.Get(obj.Token, obj.Type, obj.ID).(k8scache.Cache).Clean() } for _, obj := range diffRet.Updated { - p.caches.Get(obj.Token, obj.Type, obj.ID).(dimscache.Cache).Clean() + p.caches.Get(obj.Token, obj.Type, obj.ID).(k8scache.Cache).Clean() newCache := f.caches.Get(obj.Token, obj.Type, obj.ID) p.caches.Set(obj.Token, obj.Type, obj.ID, newCache) } for _, obj := range diffRet.Deleted { - p.caches.Get(obj.Token, obj.Type, obj.ID).(dimscache.Cache).Clean() + p.caches.Get(obj.Token, obj.Type, obj.ID).(k8scache.Cache).Clean() p.caches.Del(obj.Token, obj.Type, obj.ID) } @@ -124,7 +124,7 @@ func (p *resourceFilter) Reload(config map[string]interface{}, customized []proc func (p *resourceFilter) Clean() { for _, obj := range p.caches.All() { - obj.(dimscache.Cache).Clean() + obj.(k8scache.Cache).Clean() } } @@ -311,7 +311,7 @@ func (p *resourceFilter) replaceAction(record *define.Record, config Config) { // fromCacheAction 从缓存中补充数据 func (p *resourceFilter) fromCacheAction(record *define.Record, config Config) { token := record.Token.Original - cache := p.caches.GetByToken(token).(dimscache.Cache) + cache := p.caches.GetByToken(token).(k8scache.Cache) keys := config.FromCache.CombineKeys() handleTraces := func(resourceSpans ptrace.ResourceSpans) { @@ -325,10 +325,8 @@ func (p *resourceFilter) fromCacheAction(record *define.Record, config Config) { continue } - for _, dim := range config.FromCache.Dimensions { - if lb, ok := dims[dim]; ok { - resourceSpans.Resource().Attributes().InsertString(dim, lb) - } + for dk, dv := range dims { + resourceSpans.Resource().Attributes().InsertString(dk, dv) } return // 找到一次即可 } diff --git a/pkg/collector/processor/resourcefilter/factory_test.go b/pkg/collector/processor/resourcefilter/factory_test.go index f3899b973..28f5d4e32 100644 --- a/pkg/collector/processor/resourcefilter/factory_test.go +++ b/pkg/collector/processor/resourcefilter/factory_test.go @@ -440,18 +440,22 @@ processor: func TestTracesFromCacheAction(t *testing.T) { svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - b, _ := json.Marshal([]map[string]string{ - { - "k8s.pod.ip": "127.1.0.1", - "k8s.pod.name": "myapp1", - "k8s.namespace.name": "my-ns1", - "k8s.bcs.cluster.id": "K8S-BCS-00000", - }, - { - "k8s.pod.ip": "127.1.0.2", - "k8s.pod.name": "myapp2", - "k8s.namespace.name": "my-ns2", - "k8s.bcs.cluster.id": "K8S-BCS-90000", + b, _ := json.Marshal(map[string][]map[string]string{ + "pods": { + { + "action": "CreateOrUpdate", + "ip": "127.1.0.1", + "name": "myapp1", + "namespace": "my-ns1", + "cluster": "K8S-BCS-00000", + }, + { + "action": "CreateOrUpdate", + "ip": "127.1.0.2", + "name": "myapp2", + "namespace": "my-ns2", + "cluster": "K8S-BCS-90000", + }, }, }) w.Write(b) @@ -464,9 +468,7 @@ processor: config: from_cache: key: "resource.net.host.ip|resource.client.ip" - dimensions: ["k8s.namespace.name","k8s.pod.name","k8s.pod.ip","k8s.bcs.cluster.id"] cache: - key: "k8s.pod.ip" url: %s interval: "1m" timeout: "1m" diff --git a/pkg/collector/processor/resourcefilter/dimscache/cache.go b/pkg/collector/processor/resourcefilter/k8scache/cache.go similarity index 73% rename from pkg/collector/processor/resourcefilter/dimscache/cache.go rename to pkg/collector/processor/resourcefilter/k8scache/cache.go index c56570b9f..489894cb9 100644 --- a/pkg/collector/processor/resourcefilter/dimscache/cache.go +++ b/pkg/collector/processor/resourcefilter/k8scache/cache.go @@ -7,10 +7,11 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package dimscache +package k8scache import ( "bytes" + "fmt" "io" "net/http" "sync" @@ -25,11 +26,10 @@ type Config struct { URL string `config:"url" mapstructure:"url"` Timeout time.Duration `config:"timeout" mapstructure:"timeout"` Interval time.Duration `config:"interval" mapstructure:"interval"` - Key string `config:"key" mapstructure:"key"` } func (c *Config) Validate() bool { - if c.URL == "" || c.Key == "" { + if c.URL == "" { return false } @@ -50,8 +50,10 @@ type Cache interface { type noneCache struct{} -func (noneCache) Sync() {} +func (noneCache) Sync() {} + func (noneCache) Clean() {} + func (noneCache) Get(_ string) (map[string]string, bool) { return nil, false } @@ -62,6 +64,7 @@ type innerCache struct { conf *Config client *http.Client done chan struct{} + lastRv int synced atomic.Bool } @@ -71,8 +74,9 @@ func New(conf *Config) Cache { } tr := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: time.Minute * 5, + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: time.Minute * 5, } return &innerCache{ @@ -130,8 +134,24 @@ func (c *innerCache) Get(k string) (map[string]string, bool) { return v, ok } +type podObject struct { + Action string `json:"action"` + ClusterID string `json:"cluster"` + Name string `json:"name"` + Namespace string `json:"namespace"` + IP string `json:"ip"` +} + +type response struct { + ResourceVersion int `json:"resourceVersion"` + Pods []podObject `json:"pods"` +} + func (c *innerCache) sync() error { - req, err := http.NewRequest(http.MethodGet, c.conf.URL, &bytes.Buffer{}) + url := c.conf.URL + fmt.Sprintf("?resourceVersion=%d", c.lastRv) + logger.Debugf("innercache request url: %s", url) + + req, err := http.NewRequest(http.MethodGet, url, &bytes.Buffer{}) if err != nil { return err } @@ -148,21 +168,28 @@ func (c *innerCache) sync() error { return err } - var dims []map[string]string - if err := json.Unmarshal(buf.Bytes(), &dims); err != nil { + var ret response + if err := json.Unmarshal(buf.Bytes(), &ret); err != nil { return err } - logger.Debugf("innerCache (%s) load %d items", c.conf.URL, len(dims)) - - newCache := make(map[string]map[string]string) - for i := 0; i < len(dims); i++ { - dim := dims[i] - newCache[dim[c.conf.Key]] = dim - } + c.lastRv = ret.ResourceVersion c.mut.Lock() - c.cache = newCache - c.mut.Unlock() - + defer c.mut.Unlock() + + for _, pod := range ret.Pods { + switch pod.Action { + case "Delete": + delete(c.cache, pod.IP) + + case "CreateOrUpdate": + c.cache[pod.IP] = map[string]string{ + "k8s.bcs.cluster.id": pod.ClusterID, + "k8s.pod.name": pod.Name, + "k8s.namespace.name": pod.Namespace, + "k8s.pod.ip": pod.IP, + } + } + } return nil } diff --git a/pkg/collector/processor/resourcefilter/dimscache/cache_test.go b/pkg/collector/processor/resourcefilter/k8scache/cache_test.go similarity index 63% rename from pkg/collector/processor/resourcefilter/dimscache/cache_test.go rename to pkg/collector/processor/resourcefilter/k8scache/cache_test.go index 99db9d63f..22edfdadd 100644 --- a/pkg/collector/processor/resourcefilter/dimscache/cache_test.go +++ b/pkg/collector/processor/resourcefilter/k8scache/cache_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package dimscache +package k8scache import ( "net/http" @@ -21,27 +21,37 @@ import ( ) func TestCache(t *testing.T) { - row1 := map[string]string{ - "key1": "val1", - "key2": "val2", + rsp := response{ + ResourceVersion: 4, + Pods: []podObject{ + { + Action: "CreateOrUpdate", + ClusterID: "BCS-K8S-00000", + Name: "bkm-statefulset-worker-0", + Namespace: "bkmonitor-operator", + IP: "127.0.0.1", + }, + { + Action: "CreateOrUpdate", + ClusterID: "BCS-K8S-00000", + Name: "bkm-statefulset-worker-1", + Namespace: "bkmonitor-operator", + IP: "127.0.0.2", + }, + }, } - row2 := map[string]string{ - "key1": "val3", - "key2": "val4", - } - data := []map[string]string{row1, row2} svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - b, _ := json.Marshal(data) + b, _ := json.Marshal(rsp) w.Write(b) })) defer svr.Close() c := New(&Config{ URL: svr.URL, - Key: "key1", }) c.Sync() + defer c.Clean() // wait time.Sleep(time.Second) @@ -49,17 +59,14 @@ func TestCache(t *testing.T) { var v map[string]string var ok bool - v, ok = c.Get("val1") + v, ok = c.Get("127.0.0.1") assert.True(t, ok) - assert.Equal(t, row1, v) - - v, ok = c.Get("val2") - assert.False(t, ok) - assert.Empty(t, v) + assert.Equal(t, "bkm-statefulset-worker-0", v["k8s.pod.name"]) - v, ok = c.Get("val3") + v, ok = c.Get("127.0.0.2") assert.True(t, ok) - assert.Equal(t, row2, v) + assert.Equal(t, "bkm-statefulset-worker-1", v["k8s.pod.name"]) - c.Clean() + v, ok = c.Get("127.0.0.3") + assert.False(t, ok) } diff --git a/pkg/collector/support-files/templates/linux/x86_64/etc/bk-collector-platform.conf.tpl b/pkg/collector/support-files/templates/linux/x86_64/etc/bk-collector-platform.conf.tpl index 5805de8d1..adde30e2f 100644 --- a/pkg/collector/support-files/templates/linux/x86_64/etc/bk-collector-platform.conf.tpl +++ b/pkg/collector/support-files/templates/linux/x86_64/etc/bk-collector-platform.conf.tpl @@ -145,12 +145,7 @@ processor: {%- endfor %} from_cache: key: "{{ resource_fill_dimensions_config.from_cache.key }}" - dimensions: - {%- for dimension_key in resource_fill_dimensions_config.from_cache.dimensions %} - - "{{ dimension_key }}" - {%- endfor %} cache: - key: "{{ resource_fill_dimensions_config.from_cache.cache.key }}" url: "{{ resource_fill_dimensions_config.from_cache.cache.url }}" timeout: "{{ resource_fill_dimensions_config.from_cache.cache.timeout }}" interval: "{{ resource_fill_dimensions_config.from_cache.cache.interval }}" diff --git a/pkg/operator/operator/objectsref/controller.go b/pkg/operator/operator/objectsref/controller.go index 7e5519347..cc0edbdc6 100644 --- a/pkg/operator/operator/objectsref/controller.go +++ b/pkg/operator/operator/objectsref/controller.go @@ -34,33 +34,27 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/utils/logger" ) +type Action string + +const ( + ActionCreateOrUpdate Action = "CreateOrUpdate" + ActionDelete Action = "Delete" +) + // OwnerRef 代表 Owner 对象引用信息 type OwnerRef struct { Kind string `json:"kind"` Name string `json:"name"` } -type ContainerKey struct { - Name string - ID string - Image string -} - // Object 代表 workload 对象 type Object struct { ID ObjectID OwnerRefs []OwnerRef - // Pod 属性 - NodeName string - PodIP string - // Metadata 属性 Labels map[string]string Annotations map[string]string - - // Containers - Containers []ContainerKey } // ObjectID 代表 workload 对象标识 @@ -108,32 +102,6 @@ func (o *Objects) Del(oid ObjectID) { delete(o.objs, oid.String()) } -func (o *Objects) GetByNodeName(nodeName string) []Object { - o.mut.Lock() - defer o.mut.Unlock() - - var ret []Object - for _, obj := range o.objs { - if obj.NodeName == nodeName { - ret = append(ret, obj) - } - } - return ret -} - -func (o *Objects) GetByNamespace(namespace string) []Object { - o.mut.Lock() - defer o.mut.Unlock() - - var ret []Object - for _, obj := range o.objs { - if obj.ID.Namespace == namespace { - ret = append(ret, obj) - } - } - return ret -} - func (o *Objects) GetAll() []Object { o.mut.Lock() defer o.mut.Unlock() @@ -230,7 +198,6 @@ type ObjectsController struct { client kubernetes.Interface - podObjs *Objects replicaSetObjs *Objects deploymentObjs *Objects daemonSetObjs *Objects @@ -240,11 +207,13 @@ type ObjectsController struct { gameStatefulSetObjs *Objects gameDeploymentsObjs *Objects secretObjs *Objects - nodeObjs *NodeMap - serviceObjs *ServiceMap - endpointsObjs *EndpointsMap - ingressObjs *IngressMap - bkLogConfigObjs *BkLogConfigMap + + podObjs *PodMap + nodeObjs *NodeMap + serviceObjs *ServiceMap + endpointsObjs *EndpointsMap + ingressObjs *IngressMap + bkLogConfigObjs *BkLogConfigMap } func NewController(ctx context.Context, client kubernetes.Interface, mClient metadata.Interface, bkClient bkversioned.Interface) (*ObjectsController, error) { @@ -403,12 +372,12 @@ func (oc *ObjectsController) recordMetrics() { } } -func newPodObjects(ctx context.Context, sharedInformer informers.SharedInformerFactory) (*Objects, error) { +func newPodObjects(ctx context.Context, sharedInformer informers.SharedInformerFactory) (*PodMap, error) { genericInformer, err := sharedInformer.ForResource(corev1.SchemeGroupVersion.WithResource(resourcePods)) if err != nil { return nil, err } - objs := NewObjects(kindPod) + objs := NewPodMap() informer := genericInformer.Informer() err = informer.SetTransform(func(obj interface{}) (interface{}, error) { @@ -442,7 +411,7 @@ func newPodObjects(ctx context.Context, sharedInformer informers.SharedInformerF return } - objs.Set(Object{ + objs.Set(PodObject{ ID: ObjectID{ Name: pod.Name, Namespace: pod.Namespace, @@ -461,7 +430,8 @@ func newPodObjects(ctx context.Context, sharedInformer informers.SharedInformerF logger.Errorf("excepted Pod type, got %T", newObj) return } - objs.Set(Object{ + + objs.Set(PodObject{ ID: ObjectID{ Name: pod.Name, Namespace: pod.Namespace, @@ -1025,15 +995,3 @@ func toRefs(refs []metav1.OwnerReference) []OwnerRef { } return ret } - -func toContainerKey(pod *corev1.Pod) []ContainerKey { - var containers []ContainerKey - for _, sc := range pod.Status.ContainerStatuses { - containers = append(containers, ContainerKey{ - Name: sc.Name, - ID: sc.ContainerID, - Image: sc.ImageID, - }) - } - return containers -} diff --git a/pkg/operator/operator/objectsref/lookup.go b/pkg/operator/operator/objectsref/lookup.go index 91787d354..a62457867 100644 --- a/pkg/operator/operator/objectsref/lookup.go +++ b/pkg/operator/operator/objectsref/lookup.go @@ -9,15 +9,19 @@ package objectsref -func Lookup(oid ObjectID, objs *Objects, objsMap map[string]*Objects) *OwnerRef { +type Refer interface { + GetRefs(oid ObjectID) ([]OwnerRef, bool) +} + +func Lookup(oid ObjectID, objs Refer, objsMap map[string]*Objects) *OwnerRef { return doLookup(oid, objs, objsMap, false) } -func LookupOnce(oid ObjectID, objs *Objects, objsMap map[string]*Objects) *OwnerRef { +func LookupOnce(oid ObjectID, objs Refer, objsMap map[string]*Objects) *OwnerRef { return doLookup(oid, objs, objsMap, true) } -func doLookup(oid ObjectID, objs *Objects, objsMap map[string]*Objects, once bool) *OwnerRef { +func doLookup(oid ObjectID, objs Refer, objsMap map[string]*Objects, once bool) *OwnerRef { refs, ok := objs.GetRefs(oid) if !ok { return nil diff --git a/pkg/operator/operator/objectsref/lookup_test.go b/pkg/operator/operator/objectsref/lookup_test.go index ca2187515..884a1091f 100644 --- a/pkg/operator/operator/objectsref/lookup_test.go +++ b/pkg/operator/operator/objectsref/lookup_test.go @@ -16,7 +16,7 @@ import ( ) var ( - podObjs1 = NewObjects(kindPod) + podObjs1 = NewPodMap() replicaSetObjs1 = NewObjects(kindReplicaSet) replicaSetObjs2 = NewObjects(kindReplicaSet) deploymentObjs1 = NewObjects(kindDeployment) @@ -39,7 +39,7 @@ const ( ) func init() { - podObjs1.Set(Object{ + podObjs1.Set(PodObject{ ID: ObjectID{ Name: podName1, Namespace: namespaceDefault, @@ -53,7 +53,7 @@ func init() { NodeName: nodeName1, }) - podObjs1.Set(Object{ + podObjs1.Set(PodObject{ ID: ObjectID{ Name: podName1, Namespace: namespaceOther, @@ -67,7 +67,7 @@ func init() { NodeName: nodeName1, }) - podObjs1.Set(Object{ + podObjs1.Set(PodObject{ ID: ObjectID{ Name: podName2, Namespace: namespaceDefault, @@ -132,7 +132,7 @@ func init() { func TestLookup(t *testing.T) { cases := []struct { PodName string - PodObj *Objects + PodObj *PodMap Namespace string Refs map[string]*Objects Excepted *OwnerRef @@ -230,7 +230,7 @@ func TestLookup(t *testing.T) { func TestLookupOnce(t *testing.T) { cases := []struct { PodName string - PodObj *Objects + PodObj *PodMap Namespace string Refs map[string]*Objects Excepted *OwnerRef diff --git a/pkg/operator/operator/objectsref/pod.go b/pkg/operator/operator/objectsref/pod.go new file mode 100644 index 000000000..40ec1c00d --- /dev/null +++ b/pkg/operator/operator/objectsref/pod.go @@ -0,0 +1,193 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package objectsref + +import ( + "sync" + + corev1 "k8s.io/api/core/v1" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/operator/operator/objectsref/ring" +) + +type PodEvent struct { + Action Action + IP string + Name string + Namespace string +} + +type ContainerKey struct { + Name string + ID string + Image string +} + +type PodObject struct { + ID ObjectID + OwnerRefs []OwnerRef + NodeName string + PodIP string + Labels map[string]string + Annotations map[string]string + Containers []ContainerKey +} + +type PodMap struct { + mut sync.RWMutex + objs map[string]PodObject + ring *ring.Ring + lastRv ring.ResourceVersion +} + +func NewPodMap() *PodMap { + return &PodMap{ + objs: make(map[string]PodObject), + ring: ring.New(10240), + } +} + +func (m *PodMap) Set(obj PodObject) { + m.mut.Lock() + defer m.mut.Unlock() + + prev, ok := m.objs[obj.ID.String()] + m.objs[obj.ID.String()] = obj + + // PodIP 为空则表示该 pod 是未就绪状态 无法提供服务 + if obj.PodIP == "" { + return + } + + // 更新事件去重 + // 如果之前存在过且 PodEvent 内容均相等则不触发 ring 写入 + // 可以减少 ring 事件 因为对于 resync 时资源对象即使没有任何变更也会重新进入流程 + if ok && prev.PodIP == obj.PodIP && prev.ID == obj.ID { + return + } + + m.lastRv = m.ring.Put(PodEvent{ + Action: ActionCreateOrUpdate, + IP: obj.PodIP, + Name: obj.ID.Name, + Namespace: obj.ID.Namespace, + }) +} + +func (m *PodMap) Del(oid ObjectID) { + m.mut.Lock() + defer m.mut.Unlock() + + var podIP string + if v, ok := m.objs[oid.String()]; ok { + podIP = v.PodIP + } + + delete(m.objs, oid.String()) + m.lastRv = m.ring.Put(PodEvent{ + Action: ActionDelete, + IP: podIP, + Name: oid.Name, + Namespace: oid.Namespace, + }) +} + +func (m *PodMap) FetchEvents(rv int) ([]PodEvent, int) { + m.mut.RLock() + defer m.mut.RUnlock() + + var events []PodEvent + // fetch 所有的 pods 以事件的形式 + if rv <= 0 || rv < int(m.ring.MinResourceVersion()) { + for _, obj := range m.objs { + events = append(events, PodEvent{ + Action: ActionCreateOrUpdate, + IP: obj.PodIP, + Name: obj.ID.Name, + Namespace: obj.ID.Namespace, + }) + } + return events, int(m.lastRv) + } + + objs := m.ring.ReadGt(ring.ResourceVersion(rv)) + for _, obj := range objs { + events = append(events, obj.(PodEvent)) + } + return events, int(m.lastRv) +} + +func (m *PodMap) Counter() map[string]int { + m.mut.RLock() + defer m.mut.RUnlock() + + ret := make(map[string]int) + for _, obj := range m.objs { + ret[obj.ID.Namespace]++ + } + return ret +} + +func (m *PodMap) GetByNodeName(nodeName string) []PodObject { + m.mut.RLock() + defer m.mut.RUnlock() + + var ret []PodObject + for _, obj := range m.objs { + if obj.NodeName == nodeName { + ret = append(ret, obj) + } + } + return ret +} + +func (m *PodMap) GetByNamespace(namespace string) []PodObject { + m.mut.RLock() + defer m.mut.RUnlock() + + var ret []PodObject + for _, obj := range m.objs { + if obj.ID.Namespace == namespace { + ret = append(ret, obj) + } + } + return ret +} + +func (m *PodMap) GetAll() []PodObject { + m.mut.RLock() + defer m.mut.RUnlock() + + ret := make([]PodObject, 0, len(m.objs)) + for _, obj := range m.objs { + ret = append(ret, obj) + } + return ret +} + +func (m *PodMap) GetRefs(oid ObjectID) ([]OwnerRef, bool) { + m.mut.RLock() + defer m.mut.RUnlock() + + obj, ok := m.objs[oid.String()] + return obj.OwnerRefs, ok +} + +func toContainerKey(pod *corev1.Pod) []ContainerKey { + var containers []ContainerKey + for _, sc := range pod.Status.ContainerStatuses { + containers = append(containers, ContainerKey{ + Name: sc.Name, + ID: sc.ContainerID, + Image: sc.ImageID, + }) + } + return containers +} diff --git a/pkg/operator/operator/objectsref/controller_test.go b/pkg/operator/operator/objectsref/pod_test.go similarity index 93% rename from pkg/operator/operator/objectsref/controller_test.go rename to pkg/operator/operator/objectsref/pod_test.go index 7bb81504f..1717f7cdd 100644 --- a/pkg/operator/operator/objectsref/controller_test.go +++ b/pkg/operator/operator/objectsref/pod_test.go @@ -15,16 +15,16 @@ import ( "github.com/stretchr/testify/assert" ) -func TestObjects(t *testing.T) { - objs := NewObjects("") - objs.Set(Object{ +func TestPodMap(t *testing.T) { + objs := NewPodMap() + objs.Set(PodObject{ ID: ObjectID{ Name: "obj1", Namespace: "ns1", }, NodeName: "node1", }) - objs.Set(Object{ + objs.Set(PodObject{ ID: ObjectID{ Name: "obj2", Namespace: "ns1", diff --git a/pkg/operator/operator/objectsref/relabels.go b/pkg/operator/operator/objectsref/relabels.go index 09ab1d729..d945a4063 100644 --- a/pkg/operator/operator/objectsref/relabels.go +++ b/pkg/operator/operator/objectsref/relabels.go @@ -180,7 +180,7 @@ func (oc *ObjectsController) WorkloadsRelabelConfigsByPodName(nodeName, podName return configs } -func (oc *ObjectsController) getWorkloadRelabelConfigs(pods []Object, podName string) []RelabelConfig { +func (oc *ObjectsController) getWorkloadRelabelConfigs(pods []PodObject, podName string) []RelabelConfig { workloadRefs := make(WorkloadRefs, 0, len(pods)) for _, pod := range pods { @@ -210,7 +210,7 @@ func (oc *ObjectsController) PodsRelabelConfigs(annotations, labels []string) [] return oc.getPodRelabelConfigs(pods, "", annotations, labels) } -func (oc *ObjectsController) getPodRelabelConfigs(pods []Object, podName string, annotations, labels []string) []RelabelConfig { +func (oc *ObjectsController) getPodRelabelConfigs(pods []PodObject, podName string, annotations, labels []string) []RelabelConfig { podInfoRefs := make(PodInfoRefs, 0) parseFunc := func(s string) func(string) string { diff --git a/pkg/operator/operator/objectsref/relabels_test.go b/pkg/operator/operator/objectsref/relabels_test.go index 218f41b80..e349dacad 100644 --- a/pkg/operator/operator/objectsref/relabels_test.go +++ b/pkg/operator/operator/objectsref/relabels_test.go @@ -17,8 +17,7 @@ import ( func TestGetPodRelabelConfigs(t *testing.T) { oc := ObjectsController{} - - pods := []Object{ + pods := []PodObject{ { PodIP: "pod1", Labels: map[string]string{ diff --git a/pkg/operator/operator/objectsref/relation.go b/pkg/operator/operator/objectsref/relation.go index 0194ddd5b..91581e364 100644 --- a/pkg/operator/operator/objectsref/relation.go +++ b/pkg/operator/operator/objectsref/relation.go @@ -253,22 +253,8 @@ type StatefulSetWorker struct { Index int } -type PodInfo struct { - Name string - Namespace string - IP string -} - -func (oc *ObjectsController) AllPods() []PodInfo { - var pods []PodInfo - for _, pod := range oc.podObjs.GetAll() { - pods = append(pods, PodInfo{ - Name: pod.ID.Name, - Namespace: pod.ID.Namespace, - IP: pod.PodIP, - }) - } - return pods +func (oc *ObjectsController) FetchPodEvents(rv int) ([]PodEvent, int) { + return oc.podObjs.FetchEvents(rv) } func (oc *ObjectsController) GetPods(s string) map[string]StatefulSetWorker { diff --git a/pkg/operator/operator/objectsref/relation_test.go b/pkg/operator/operator/objectsref/relation_test.go index c62d88e91..22b1fce1f 100644 --- a/pkg/operator/operator/objectsref/relation_test.go +++ b/pkg/operator/operator/objectsref/relation_test.go @@ -26,7 +26,7 @@ func TestWritePodRelations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - podObject := Object{ + podObject := PodObject{ ID: ObjectID{ Name: "test-pod-1", Namespace: "test-ns-1", @@ -41,9 +41,8 @@ func TestWritePodRelations(t *testing.T) { objectsController := &ObjectsController{ ctx: ctx, cancel: cancel, - podObjs: &Objects{ - kind: kindPod, - objs: map[string]Object{ + podObjs: &PodMap{ + objs: map[string]PodObject{ podObject.ID.String(): podObject, }, }, @@ -63,7 +62,7 @@ func TestWriteDataSourceRelations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pods := []Object{ + pods := []PodObject{ { ID: ObjectID{ Name: "unify-query-01", @@ -101,7 +100,7 @@ func TestWriteDataSourceRelations(t *testing.T) { Containers: []ContainerKey{{Name: "unify-query"}}, }, } - podsMap := make(map[string]Object, len(pods)) + podsMap := make(map[string]PodObject, len(pods)) for _, p := range pods { podsMap[p.ID.String()] = p } @@ -262,8 +261,7 @@ func TestWriteDataSourceRelations(t *testing.T) { nodeObjs: &NodeMap{ nodes: nodesMap, }, - podObjs: &Objects{ - kind: kindPod, + podObjs: &PodMap{ objs: podsMap, }, } diff --git a/pkg/operator/operator/objectsref/ring/ring.go b/pkg/operator/operator/objectsref/ring/ring.go new file mode 100644 index 000000000..5faf94f73 --- /dev/null +++ b/pkg/operator/operator/objectsref/ring/ring.go @@ -0,0 +1,140 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package ring + +import ( + "sync" +) + +// ResourceVersion 事件版次号 ring 实例内自增如果有变更操作 +type ResourceVersion int + +type Ring struct { + ring *ring +} + +// New 构建 *Ring 实例 +// size 为 ring 容量 写操作为环形覆写 同时每次都会记录 head/tail 所在 index +// 操作为线程安全 +func New(size int) *Ring { + return &Ring{ring: newRing(size)} +} + +type event struct { + resourceVersion int + obj interface{} +} + +type ring struct { + mut sync.RWMutex + size int + maxRv int + minRv int + headIdx int + tailIdx int + events []event +} + +func newRing(size int) *ring { + events := make([]event, size) + return &ring{ + size: size, + events: events, + } +} + +func (r *ring) put(evt event) int { + r.mut.Lock() + defer r.mut.Unlock() + + // 双游标方案 + // headIndex 记录 ring 内当前最早的事件 index 位置 + // tailIndex 记录 ring 内当前最新的事件 index 位置 + tailIdx := r.maxRv % r.size + var headIdx int + if r.maxRv >= r.size { + headIdx = (tailIdx + 1) % r.size + } + + r.maxRv++ + r.tailIdx = tailIdx + r.headIdx = headIdx + + evt.resourceVersion = r.maxRv + r.events[r.tailIdx] = evt + r.minRv = r.events[r.headIdx].resourceVersion + return r.maxRv +} + +func (r *ring) readGt(n int) []event { + r.mut.RLock() + defer r.mut.RUnlock() + + var events []event + + // head 遍历为左闭右开区间 + for i := r.headIdx; i < r.size; i++ { + evt := r.events[i] + if evt.resourceVersion > n { + events = append(events, evt) + } + } + + // headIndex 非 0 则意味着需要反向遍历一遍 + // tail 遍历为闭区间 + if r.headIdx != 0 { + for i := 0; i <= r.tailIdx; i++ { + evt := r.events[i] + if evt.resourceVersion > n { + events = append(events, evt) + } + } + } + return events +} + +func (r *ring) minResourceVersion() int { + r.mut.RLock() + defer r.mut.RUnlock() + + return r.minRv +} + +func (r *ring) maxResourceVersion() int { + r.mut.RLock() + defer r.mut.RUnlock() + + return r.maxRv +} + +// Put 将 obj 存放至环内 同时返回当前最新版次号 +func (q *Ring) Put(obj interface{}) ResourceVersion { + return ResourceVersion(q.ring.put(event{ + obj: obj, + })) +} + +// ReadGt 读取 > rv 的所有资源对象 +func (q *Ring) ReadGt(rv ResourceVersion) []interface{} { + events := q.ring.readGt(int(rv)) + objs := make([]interface{}, 0, len(events)) + for _, evt := range events { + objs = append(objs, evt.obj) + } + return objs +} + +func (q *Ring) MinResourceVersion() ResourceVersion { + return ResourceVersion(q.ring.minResourceVersion()) +} + +func (q *Ring) MaxResourceVersion() ResourceVersion { + return ResourceVersion(q.ring.maxResourceVersion()) +} diff --git a/pkg/operator/operator/objectsref/ring/ring_test.go b/pkg/operator/operator/objectsref/ring/ring_test.go new file mode 100644 index 000000000..8e25a5b8c --- /dev/null +++ b/pkg/operator/operator/objectsref/ring/ring_test.go @@ -0,0 +1,77 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package ring + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRing(t *testing.T) { + t.Run("ring 1", func(t *testing.T) { + q := New(5) + rv := q.Put(strconv.Itoa(0)) + + assert.Equal(t, ResourceVersion(1), rv) + assert.Equal(t, ResourceVersion(1), q.MinResourceVersion()) + assert.Equal(t, ResourceVersion(1), q.MaxResourceVersion()) + + objs := []interface{}{"0"} + assert.Equal(t, objs, q.ReadGt(0)) + }) + + t.Run("ring 2", func(t *testing.T) { + q := New(5) + var rv ResourceVersion + for i := 0; i < 5; i++ { + rv = q.Put(strconv.Itoa(i)) + } + + assert.Equal(t, ResourceVersion(5), rv) + assert.Equal(t, ResourceVersion(1), q.MinResourceVersion()) + assert.Equal(t, ResourceVersion(5), q.MaxResourceVersion()) + + objs := []interface{}{"0", "1", "2", "3", "4"} + assert.Equal(t, objs, q.ReadGt(0)) + }) + + t.Run("ring with start index", func(t *testing.T) { + q := New(5) + for i := 0; i < 5; i++ { + q.Put(strconv.Itoa(i)) + } + objs := []interface{}{"3", "4"} + assert.Equal(t, objs, q.ReadGt(3)) + }) + + t.Run("ring oversize", func(t *testing.T) { + q := New(5) + for i := 0; i < 7; i++ { + q.Put(strconv.Itoa(i)) + } + + assert.Equal(t, ResourceVersion(3), q.MinResourceVersion()) + assert.Equal(t, ResourceVersion(7), q.MaxResourceVersion()) + + objs := []interface{}{"2", "3", "4", "5", "6"} + assert.Equal(t, objs, q.ReadGt(0)) + }) + + t.Run("ring oversize with start index", func(t *testing.T) { + q := New(5) + for i := 0; i < 7; i++ { + q.Put(strconv.Itoa(i)) + } + objs := []interface{}{"5", "6"} + assert.Equal(t, objs, q.ReadGt(5)) + }) +} diff --git a/pkg/operator/operator/server.go b/pkg/operator/operator/server.go index 2a2f5d94b..e3d59d35c 100644 --- a/pkg/operator/operator/server.go +++ b/pkg/operator/operator/server.go @@ -481,7 +481,6 @@ func (c *Operator) WorkloadRoute(w http.ResponseWriter, _ *http.Request) { } func (c *Operator) PodsRoute(w http.ResponseWriter, r *http.Request) { - pods := c.objectsController.AllPods() info, err := c.dw.GetClusterInfo() if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -489,29 +488,40 @@ func (c *Operator) PodsRoute(w http.ResponseWriter, r *http.Request) { return } + // 无此参数默认按 0 处理 + rv, _ := strconv.Atoi(r.URL.Query().Get("resourceVersion")) type podsResponse struct { - ClusterID string `json:"k8s.bcs.cluster.id"` - Name string `json:"k8s.pod.name"` - Namespace string `json:"k8s.namespace.name"` - IP string `json:"k8s.pod.ip"` + Action string `json:"action"` + ClusterID string `json:"cluster"` + Name string `json:"name"` + Namespace string `json:"namespace"` + IP string `json:"ip"` } - nodes := c.objectsController.NodeIPs() - all := r.URL.Query().Get("all") + all := r.URL.Query().Get("all") // all 则返回所有 pods 不进行任何过滤 + // 只返回已经就绪的 Pod + podEvents, lastRv := c.objectsController.FetchPodEvents(rv) + nodes := c.objectsController.NodeIPs() var ret []podsResponse - for _, pod := range pods { - _, ok := nodes[pod.IP] + for _, podEvent := range podEvents { + _, ok := nodes[podEvent.IP] if !ok || all == "true" { ret = append(ret, podsResponse{ + Action: string(podEvent.Action), ClusterID: info.BcsClusterID, - Name: pod.Name, - Namespace: pod.Namespace, - IP: pod.IP, + Name: podEvent.Name, + Namespace: podEvent.Namespace, + IP: podEvent.IP, }) } } - writeResponse(w, ret) + + type R struct { + Pods []podsResponse `json:"pods"` + ResourceVersion int `json:"resourceVersion"` + } + writeResponse(w, R{Pods: ret, ResourceVersion: lastRv}) } func (c *Operator) WorkloadNodeRoute(w http.ResponseWriter, r *http.Request) { @@ -598,7 +608,7 @@ func (c *Operator) IndexRoute(w http.ResponseWriter, _ *http.Request) { * GET /cluster_info * GET /workload * GET /workload/node/{node} -* GET /pods?all=true|false +* GET /pods?resourceVersion=N&all=true|false * GET /relation/metrics * GET /rule/metrics * GET /configs diff --git a/pkg/unify-query/internal/querystring/expr.go b/pkg/unify-query/internal/querystring/expr.go index b9e8ea175..c070dbbc7 100644 --- a/pkg/unify-query/internal/querystring/expr.go +++ b/pkg/unify-query/internal/querystring/expr.go @@ -172,3 +172,8 @@ func newStringExpr(str string) FieldableExpr { return NewMatchExpr(str) } + +type TermExpr struct { + Field string + Value string +} diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 45570fe2c..1081d8eec 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -29,6 +29,24 @@ const ( UUID = "query_uuid" ) +const ( + ConditionEqual = "eq" + ConditionNotEqual = "ne" + ConditionRegEqual = "req" + ConditionNotRegEqual = "nreq" + ConditionContains = "contains" + ConditionNotContains = "ncontains" + + ConditionExisted = "existed" + ConditionNotExisted = "nexisted" + + ConditionExact = "exact" + ConditionGt = "gt" + ConditionGte = "gte" + ConditionLt = "lt" + ConditionLte = "lte" +) + type VmCondition string type TimeField struct { @@ -40,7 +58,8 @@ type TimeField struct { // Aggregate 聚合方法 type Aggregate struct { - Name string + Name string + Dimensions []string Without bool diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index bd02d3a1a..8df8cc057 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -564,6 +564,26 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q MetricName: metricName, Aggregates: aggregates, BkSqlCondition: allConditions.BkSql(), + Size: q.Limit, + From: q.From, + } + + if len(q.OrderBy) > 0 { + qry.Orders = make(metadata.Orders) + for _, o := range q.OrderBy { + if len(o) == 0 { + continue + } + + asc := true + name := o + + if strings.HasPrefix(o, "-") { + asc = false + name = name[1:] + } + qry.Orders[name] = asc + } } metadata.GetQueryParams(ctx).SetStorageType(qry.StorageType) @@ -872,21 +892,19 @@ func (q *Query) BuildMetadataQuery( return nil, err } - // 在 metadata 还没有补充 storageType 字段之前 - // 使用 sourceType 来判断是否是 es 查询 - // 等后面支持了之后可以删除该段逻辑 - if tsDB.SourceType == BkData { - query.StorageType = consul.ElasticsearchStorageType - } - // 兼容原逻辑，storageType 通过 storageMap 获取 if query.StorageType == "" { + log.Warnf(ctx, "storageType is empty with %s", query.TableID) stg, _ := tsdb.GetStorage(query.StorageID) if stg != nil { query.StorageType = stg.Type } } + if query.StorageType == "" { + return nil, fmt.Errorf("storageType is empty with %d", query.StorageID) + } + query.Measurement = measurement query.Measurements = measurements query.Field = field diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 7e9a776f9..e3f6063ea 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -285,11 +285,11 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot return } -func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs) (*PromData, error) { +func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.QueryTs) (*PromData, error) { var ( res any err error - resp = NewPromData(query.ResultColumns) + resp = NewPromData(queryTs.ResultColumns) ) ctx, span := trace.NewSpan(ctx, "query-reference") @@ -299,35 +299,52 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs span.End(&err) }() - qStr, _ := json.Marshal(query) + qStr, _ := json.Marshal(queryTs) span.Set("query-ts", string(qStr)) - for _, q := range query.QueryList { - q.IsReference = true - if q.TableID == "" { + for _, ql := range queryTs.QueryList { + ql.IsReference = true + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } + + if ql.TableID == "" { err = fmt.Errorf("tableID is empty") return nil, err } } // 判断如果 step 为空，则补充默认 step - if query.Step == "" { - query.Step = promql.GetDefaultStep().String() + if queryTs.Step == "" { + queryTs.Step = promql.GetDefaultStep().String() } - queryRef, err := query.ToQueryReference(ctx) - startInt, err := strconv.ParseInt(query.Start, 10, 64) + queryRef, err := queryTs.ToQueryReference(ctx) + startInt, err := strconv.ParseInt(queryTs.Start, 10, 64) if err != nil { return nil, err } start := time.Unix(startInt, 0) - endInt, err := strconv.ParseInt(query.End, 10, 64) + endInt, err := strconv.ParseInt(queryTs.End, 10, 64) if err != nil { return nil, err } end := time.Unix(endInt, 0) - step, err := model.ParseDuration(query.Step) + step, err := model.ParseDuration(queryTs.Step) if err != nil { return nil, err } @@ -337,8 +354,8 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs metadata.SetQueryReference(ctx, queryRef) var lookBackDelta time.Duration - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return nil, err } @@ -349,10 +366,10 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs Timeout: SingleflightTimeout, }, lookBackDelta, QueryMaxRouting) - if query.Instant { - res, err = instance.DirectQuery(ctx, query.MetricMerge, start) + if queryTs.Instant { + res, err = instance.DirectQuery(ctx, queryTs.MetricMerge, start) } else { - res, err = instance.DirectQueryRange(ctx, query.MetricMerge, start, end, time.Duration(step)) + res, err = instance.DirectQueryRange(ctx, queryTs.MetricMerge, start, end, time.Duration(step)) } if err != nil { return nil, err @@ -397,7 +414,7 @@ func queryReferenceWithPromEngine(ctx context.Context, query *structured.QueryTs } // queryTsToInstanceAndStmt query 结构体转换为 instance 以及 stmt -func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { +func queryTsToInstanceAndStmt(ctx context.Context, queryTs *structured.QueryTs) (instance tsdb.Instance, stmt string, err error) { var ( lookBackDelta time.Duration promExprOpt = &structured.PromExprOption{} @@ -408,37 +425,53 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i span.End(&err) }() - queryString, _ := json.Marshal(query) + queryString, _ := json.Marshal(queryTs) span.Set("query-ts", queryString) // 限制 queryList 是否过长 if DefaultQueryListLimit > 0 { - if len(query.QueryList) > DefaultQueryListLimit { + if len(queryTs.QueryList) > DefaultQueryListLimit { err = fmt.Errorf("the number of query lists cannot be greater than %d", DefaultQueryListLimit) } } // 判断是否打开对齐 - for _, q := range query.QueryList { - q.IsReference = false - q.AlignInfluxdbResult = AlignInfluxdbResult + for _, ql := range queryTs.QueryList { + ql.IsReference = false + ql.AlignInfluxdbResult = AlignInfluxdbResult + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } } // 判断是否指定 LookBackDelta - if query.LookBackDelta != "" { - lookBackDelta, err = time.ParseDuration(query.LookBackDelta) + if queryTs.LookBackDelta != "" { + lookBackDelta, err = time.ParseDuration(queryTs.LookBackDelta) if err != nil { return } } // 如果 step 为空，则补充默认 step - if query.Step == "" { - query.Step = promql.GetDefaultStep().String() + if queryTs.Step == "" { + queryTs.Step = promql.GetDefaultStep().String() } // 转换成 queryRef - queryRef, err := query.ToQueryReference(ctx) + queryRef, err := queryTs.ToQueryReference(ctx) if err != nil { return } @@ -467,7 +500,7 @@ func queryTsToInstanceAndStmt(ctx context.Context, query *structured.QueryTs) (i }, lookBackDelta, QueryMaxRouting) } - expr, err := query.ToPromExpr(ctx, promExprOpt) + expr, err := queryTs.ToPromExpr(ctx, promExprOpt) if err != nil { return } diff --git a/pkg/unify-query/service/influxdb/hook.go b/pkg/unify-query/service/influxdb/hook.go index fa4521b7a..ec681fad3 100644 --- a/pkg/unify-query/service/influxdb/hook.go +++ b/pkg/unify-query/service/influxdb/hook.go @@ -45,6 +45,8 @@ func setDefaultConfig() { viper.SetDefault(GrpcMaxCallRecvMsgSizeConfigPath, 1024*1024*10) viper.SetDefault(GrpcMaxCallSendMsgSizeConfigPath, 1024*1024*10) + + viper.SetDefault(IsCacheConfigPath, true) } // LoadConfig @@ -71,6 +73,7 @@ func LoadConfig() { SpaceRouterBboltPath = viper.GetString(SpaceRouterBboltPathConfigPath) SpaceRouterBboltBucketName = viper.GetString(SpaceRouterBboltBucketNameConfigPath) SpaceRouterBboltWriteBatchSize = viper.GetInt(SpaceRouterBboltWriteBatchSizeConfigPath) + IsCache = viper.GetBool(IsCacheConfigPath) GrpcMaxCallRecvMsgSize = viper.GetInt(GrpcMaxCallRecvMsgSizeConfigPath) GrpcMaxCallSendMsgSize = viper.GetInt(GrpcMaxCallSendMsgSizeConfigPath) diff --git a/pkg/unify-query/service/influxdb/service.go b/pkg/unify-query/service/influxdb/service.go index f8b41cf12..39bffbf4d 100644 --- a/pkg/unify-query/service/influxdb/service.go +++ b/pkg/unify-query/service/influxdb/service.go @@ -69,19 +69,16 @@ func (s *Service) Reload(ctx context.Context) { err = s.loopReloadStorage(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload es storage failed for->[%s]", err) - return } err = s.loopReloadTableInfo(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload table info failed,error:%s", err) - return } err = s.loopReloadRouter(s.ctx) if err != nil { log.Errorf(context.TODO(), "start loop reload query router failed,error:%s", err) - return } err = s.loopReloadBCSInfo(s.ctx) @@ -313,7 +310,7 @@ func (s *Service) reloadInfluxDBRouter(ctx context.Context) error { // reloadInfluxDBRouter 重新加载 SpaceTsDbRouter func (s *Service) reloadSpaceTsDbRouter(ctx context.Context) error { - ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, true) + ir, err := inner.SetSpaceTsDbRouter(ctx, SpaceRouterBboltPath, SpaceRouterBboltBucketName, SpaceRouterPrefix, SpaceRouterBboltWriteBatchSize, IsCache) if err != nil { return err } diff --git a/pkg/unify-query/service/influxdb/settings.go b/pkg/unify-query/service/influxdb/settings.go index 95fe57921..430c7dd15 100644 --- a/pkg/unify-query/service/influxdb/settings.go +++ b/pkg/unify-query/service/influxdb/settings.go @@ -37,6 +37,8 @@ const ( GrpcMaxCallRecvMsgSizeConfigPath = "influxdb.grpc_max_call_recv_msg_size" GrpcMaxCallSendMsgSizeConfigPath = "influxdb.grpc_max_call_send_msg_size" + + IsCacheConfigPath = "influxdb.is_cache" ) var ( @@ -61,6 +63,8 @@ var ( SpaceRouterBboltBucketName string SpaceRouterBboltWriteBatchSize int + IsCache bool + GrpcMaxCallRecvMsgSize int GrpcMaxCallSendMsgSize int ) diff --git a/pkg/unify-query/tsdb/bksql/client_test.go b/pkg/unify-query/tsdb/bksql/client_test.go index 8761c7358..593bb0a52 100644 --- a/pkg/unify-query/tsdb/bksql/client_test.go +++ b/pkg/unify-query/tsdb/bksql/client_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -21,15 +21,16 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) var ( - client *Client + client *bksql.Client ) -func MockClient() *Client { +func MockClient() *bksql.Client { if client == nil { - client = (&Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) + client = (&bksql.Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) } return client @@ -43,10 +44,10 @@ func TestClient_QuerySync(t *testing.T) { end := time.UnixMilli(1729838923416) mock.BkSQL.Set(map[string]any{ - `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &Result{ + `SELECT * FROM restriction_table WHERE dtEventTimeStamp >= 1729838623416 AND dtEventTimeStamp < 1729838923416 LIMIT 5`: &bksql.Result{ Result: true, - Code: StatusOK, - Data: &QuerySyncResultData{ + Code: bksql.StatusOK, + Data: &bksql.QuerySyncResultData{ TotalRecords: 5, SelectFieldsOrder: []string{ "dtEventTimeStamp", @@ -94,8 +95,8 @@ func TestClient_QuerySync(t *testing.T) { nil, ) - assert.Equal(t, StatusOK, res.Code) - d, ok := res.Data.(*QuerySyncResultData) + assert.Equal(t, bksql.StatusOK, res.Code) + d, ok := res.Data.(*bksql.QuerySyncResultData) assert.True(t, ok) assert.Equal(t, d.TotalRecords, 5) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 04433502c..9cb910057 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -21,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) const ( @@ -132,7 +133,7 @@ func (f *QueryFactory) ParserQuery() (err error) { return } -func (f *QueryFactory) getTheDateFilters() (theDateFilter string, err error) { +func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { @@ -157,6 +158,48 @@ func (f *QueryFactory) getTheDateFilters() (theDateFilter string, err error) { return } +func (f *QueryFactory) BuildWhere() (string, error) { + var s []string + s = append(s, fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) + + theDateFilter, err := f.getTheDateIndexFilters() + if err != nil { + return "", err + } + if theDateFilter != "" { + s = append(s, theDateFilter) + } + + fieldsMap := make(map[string]string) + expr := sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap) + + // QueryString to sql + if f.query.QueryString != "" { + qs, err := expr.ParserQueryString(f.query.QueryString) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + // AllConditions to sql + if len(f.query.AllConditions) > 0 { + qs, err := expr.ParserAllConditions(f.query.AllConditions) + if err != nil { + return "", err + } + + if qs != "" { + s = append(s, qs) + } + } + + return strings.Join(s, " AND "), nil +} + func (f *QueryFactory) SQL() (sql string, err error) { f.sql.Reset() err = f.ParserQuery() @@ -175,22 +218,16 @@ func (f *QueryFactory) SQL() (sql string, err error) { db += "." + f.query.Measurement } f.write(db) - f.write("WHERE") - f.write(fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) - theDateFilter, err := f.getTheDateFilters() + where, err := f.BuildWhere() if err != nil { return } - if theDateFilter != "" { - f.write("AND") - f.write(theDateFilter) + if where != "" { + f.write("WHERE") + f.write(where) } - if f.query.BkSqlCondition != "" { - f.write("AND") - f.write("(" + f.query.BkSqlCondition + ")") - } if len(f.groups) > 0 { f.write("GROUP BY") f.write(strings.Join(f.groups, ", ")) diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index b2f0ba189..7f0a98383 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -18,6 +18,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) func TestNewSqlFactory(t *testing.T) { @@ -45,12 +46,20 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 0, - Orders: metadata.Orders{"ip": true}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + }, + }, + From: 0, + Size: 0, + Orders: metadata.Orders{"ip": true}, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", }, "sum-with-promql-1": { query: &metadata.Query{ @@ -65,12 +74,20 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - BkSqlCondition: "gseIndex > 0", - From: 0, - Size: 10, - Orders: nil, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + }, + }, + From: 0, + Size: 10, + Orders: nil, }, - expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip` LIMIT 10", + expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, "count-with-count-promql-1": { query: &metadata.Query{ @@ -86,9 +103,8 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, "count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -110,9 +126,17 @@ func TestNewSqlFactory(t *testing.T) { Window: time.Minute, }, }, - BkSqlCondition: "gseIndex > 0", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "gseIndex", + Value: []string{"0"}, + Operator: metadata.ConditionGt, + }, + }, + }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND (gseIndex > 0) GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { @@ -125,7 +149,7 @@ func TestNewSqlFactory(t *testing.T) { } log.Infof(ctx, "start: %s, end: %s", c.start, c.end) - fact := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) sql, err := fact.SQL() assert.Nil(t, err) assert.Equal(t, c.expected, sql) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 28e6d0827..91867371a 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -31,6 +31,14 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" ) +const ( + KeyHighLight = "__highlight" + + KeyIndex = "__index" + KeyTableID = "__result_table" + KeyDataLabel = "__data_label" +) + type Instance struct { ctx context.Context @@ -48,11 +56,6 @@ type Instance struct { client *Client } -func (i *Instance) QueryReference(ctx context.Context, query *metadata.Query, start int64, end int64) (*prompb.QueryResult, error) { - //TODO implement me - panic("implement me") -} - var _ tsdb.Instance = (*Instance)(nil) type Options struct { @@ -275,15 +278,79 @@ func (i *Instance) table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("doris query error: %s", r) + } + }() + + ctx, span := trace.NewSpan(ctx, "bk-sql-query-raw") + defer span.End(&err) + + span.Set("query-raw-start", start) + span.Set("query-raw-end", end) + + if start.UnixMilli() > end.UnixMilli() || start.UnixMilli() == 0 { + return + } + + rangeLeftTime := end.Sub(start) + metric.TsDBRequestRangeMinute(ctx, rangeLeftTime, i.InstanceType()) + + if i.maxLimit > 0 { + maxLimit := i.maxLimit + i.tolerance + // 如果不传 size，则取最大的限制值 + if query.Size == 0 || query.Size > i.maxLimit { + query.Size = maxLimit + } + } + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + + sql, err := queryFactory.SQL() + if err != nil { + return + } + + data, err := i.sqlQuery(ctx, sql, span) + if err != nil { + return + } + + if data == nil { + return + } + + span.Set("data-total-records", data.TotalRecords) + log.Infof(ctx, "total records: %d", data.TotalRecords) + + if i.maxLimit > 0 && data.TotalRecords > i.maxLimit { + return + } + + for _, list := range data.List { + list[KeyIndex] = query.DB + list[KeyTableID] = query.TableID + list[KeyDataLabel] = query.DataLabel + list[FieldTime] = list[timeStamp] + + if query.HighLight.Enable { + list[KeyHighLight] = "" + } + + dataCh <- list + } + + total = int64(data.TotalRecords) + return } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { var ( err error ) - ctx, span := trace.NewSpan(ctx, "bk-sql-raw") + ctx, span := trace.NewSpan(ctx, "bk-sql-query-series-set") defer span.End(&err) span.Set("query-series-set-start", start) @@ -336,18 +403,15 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } func (i *Instance) DirectQueryRange(ctx context.Context, promql string, start, end time.Time, step time.Duration) (promql.Matrix, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectQuery(ctx context.Context, qs string, end time.Time) (promql.Vector, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, start, end time.Time) ([]string, error) { @@ -418,18 +482,15 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, } func (i *Instance) QuerySeries(ctx context.Context, query *metadata.Query, start, end time.Time) ([]map[string]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelNames(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelValues(ctx context.Context, name string, start, end time.Time, limit int, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) InstanceType() string { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 568125764..e169bdcfc 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package bksql +package bksql_test import ( "context" @@ -21,6 +21,8 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) func TestInstance_QueryRaw(t *testing.T) { @@ -28,7 +30,7 @@ func TestInstance_QueryRaw(t *testing.T) { ctx := context.Background() mock.Init() - ins, err := NewInstance(ctx, &Options{ + ins, err := bksql.NewInstance(ctx, &bksql.Options{ Address: mock.BkSQLUrl, Timeout: time.Minute, MaxLimit: 1e4, @@ -41,7 +43,7 @@ func TestInstance_QueryRaw(t *testing.T) { } mock.BkSQL.Set(map[string]any{ - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND (namespace = 'gz100' OR namespace = 'bgp2\\-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND ((namespace = 'gz100') OR (namespace = 'bgp2-new')) LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) @@ -64,8 +66,17 @@ func TestInstance_QueryRaw(t *testing.T) { TableID: tableID, DB: db, MetricName: field, - BkSqlCondition: "namespace = 'gz100' OR namespace = 'bgp2\\-new'", + BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", OffsetInfo: metadata.OffSetInfo{Limit: 10}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "namespace", + Operator: metadata.ConditionEqual, + Value: []string{"gz100", "bgp2-new"}, + }, + }, + }, }, expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":269,"timestamp":1730118660000},{"value":271,"timestamp":1730118480000},{"value":267,"timestamp":1730118540000},{"value":274,"timestamp":1730118600000},{"value":279,"timestamp":1730118420000}],"exemplars":null,"histograms":null}]`, }, @@ -136,7 +147,7 @@ func TestInstance_bkSql(t *testing.T) { query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", - BkSqlCondition: "namespace REGEXP '^(bgp2\\-new|gz100)$'", + BkSqlCondition: "`namespace` IN ('bgp2-new', 'gz100')", Aggregates: metadata.Aggregates{ { Name: "count", @@ -144,8 +155,17 @@ func TestInstance_bkSql(t *testing.T) { Window: time.Second * 15, }, }, + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"bgp2-new", "gz100"}, + }, + }, + }, }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (namespace REGEXP '^(bgp2\\-new|gz100)$') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", }, { query: &metadata.Query{ @@ -241,7 +261,14 @@ func TestInstance_bkSql(t *testing.T) { c.end = end } - sql, err := NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end).SQL() + condition, err := sqlExpr.GetSQLExpr(c.query.Measurement).WithFieldsMap(nil).ParserAllConditions(c.query.AllConditions) + assert.Nil(t, err) + if err == nil { + assert.Equal(t, c.query.BkSqlCondition, condition) + } + + fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + sql, err := fact.SQL() assert.Nil(t, err) if err == nil { assert.Equal(t, c.expected, sql) @@ -249,3 +276,170 @@ func TestInstance_bkSql(t *testing.T) { }) } } + +func TestInstance_bkSql_EdgeCases(t *testing.T) { + mock.Init() + + // 基础时间范围 + baseStart := time.UnixMilli(1718189940000) + baseEnd := time.UnixMilli(1718193555000) + + // 跨天时间范围 + crossDayStart := time.Unix(1733756400, 0) // 2024-12-09 00:00:00 + crossDayEnd := time.Unix(1733846399, 0) // 2024-12-09 23:59:59 + + testCases := []struct { + name string + start time.Time + end time.Time + query *metadata.Query + expected string + }{ + // 测试用例1: 无聚合函数的原始查询 + { + name: "raw query without aggregation", + query: &metadata.Query{ + DB: "test_db", + Field: "value", + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `test_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例2: 多聚合函数组合 + { + name: "multiple aggregates", + query: &metadata.Query{ + DB: "metrics_db", + Field: "temperature", + Aggregates: metadata.Aggregates{ + {Name: "max"}, + {Name: "min"}, + }, + }, + expected: "SELECT MAX(`temperature`) AS `_value_`, MIN(`temperature`) AS `_value_` FROM `metrics_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例3: 复杂条件组合 + { + name: "complex conditions", + query: &metadata.Query{ + DB: "security_logs", + Field: "duration", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "severity", + Operator: metadata.ConditionEqual, + Value: []string{"high", "critical"}, + }, + { + DimensionName: "source_ip", + Operator: metadata.ConditionNotContains, + Value: []string{"192.168.1.1"}, + }, + }, + }, + }, + expected: "SELECT *, `duration` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `security_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `severity` IN ('high', 'critical') AND `source_ip` != '192.168.1.1'", + }, + + // 测试用例4: 多字段排序 + { + name: "multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "amount", + Orders: map[string]bool{ + "timestamp": true, + "account_id": false, + }, + }, + expected: "SELECT *, `amount` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `account_id` DESC, `timestamp` ASC", + }, + + // 测试用例5: 特殊字符转义 + { + name: "special characters in fields", + query: &metadata.Query{ + DB: "special_metrics", + Measurement: "select", // 保留字作为measurement + Field: "*", + Aggregates: metadata.Aggregates{ + {Name: "sum"}, + }, + }, + expected: "SELECT SUM(`*`) AS `_value_` FROM `special_metrics`.select WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例6: 零窗口时间 + { + name: "zero window size", + query: &metadata.Query{ + DB: "time_series_data", + Field: "value", + Aggregates: metadata.Aggregates{ + { + Name: "avg", + Window: 0, + }, + }, + }, + expected: "SELECT AVG(`value`) AS `_value_` FROM `time_series_data` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + + // 测试用例7: 跨多天的时间范围 + { + name: "multi-day time range", + start: crossDayStart, + end: crossDayEnd, + query: &metadata.Query{ + DB: "daily_metrics", + Field: "active_users", + Aggregates: metadata.Aggregates{ + {Name: "count"}, + }, + }, + expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + + // 设置默认时间范围 + start := tc.start + if start.IsZero() { + start = baseStart + } + end := tc.end + if end.IsZero() { + end = baseEnd + } + + // 条件解析验证 + if len(tc.query.AllConditions) > 0 { + condition, err := sqlExpr.GetSQLExpr(tc.query.Measurement).WithFieldsMap(nil).ParserAllConditions(tc.query.AllConditions) + assert.Nil(t, err) + if err == nil { + assert.NotEmpty(t, condition, "Parsed condition should not be empty") + } + } + + // SQL生成验证 + fact := bksql.NewQueryFactory(ctx, tc.query).WithRangeTime(start, end) + generatedSQL, err := fact.SQL() + + assert.Nil(t, err) + if err == nil { + assert.Equal(t, tc.expected, generatedSQL) + } + + // 验证时间条件 + if tc.start.IsZero() && tc.end.IsZero() { + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` < %d", baseEnd.UnixMilli())) + } + }) + } +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go new file mode 100644 index 000000000..fc8a9bdc7 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -0,0 +1,240 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr + +import ( + "fmt" + "strings" + "sync" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" +) + +// ErrorMatchAll 定义全字段检索错误提示信息 +var ( + ErrorMatchAll = "不支持全字段检索" +) + +// SQLExpr 定义SQL表达式生成接口 +// 接口包含字段映射、查询字符串解析、全条件解析等功能 +type SQLExpr interface { + // WithFieldsMap 设置字段类型 + WithFieldsMap(fieldsMap map[string]string) SQLExpr + // WithTransformDimension 设置字段转换方法 + WithTransformDimension(func(string) string) SQLExpr + // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 + ParserQueryString(qs string) (string, error) + // ParserAllConditions 解析全量条件生成SQL条件表达式 + ParserAllConditions(allConditions metadata.AllConditions) (string, error) +} + +// SQL表达式注册管理相关变量 +var ( + _ SQLExpr = (*DefaultSQLExpr)(nil) // 接口实现检查 + + lock sync.RWMutex // 读写锁用于并发安全 + ExprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 +) + +// GetSQLExpr 获取指定key的SQL表达式实现 +// 参数： +// +// key - 注册时使用的标识符 +// +// 返回值： +// +// 找到返回对应实现，未找到返回默认实现 +func GetSQLExpr(key string) SQLExpr { + lock.RLock() + defer lock.RUnlock() + if sqlExpr, ok := ExprMap[key]; ok { + return sqlExpr + } else { + return &DefaultSQLExpr{} + } +} + +// Register 注册SQL表达式实现 +// 参数： +// +// key - 实现标识符 +// sqlExpr - 具体的SQL表达式实现 +func Register(key string, sqlExpr SQLExpr) { + lock.Lock() + defer lock.Unlock() + ExprMap[key] = sqlExpr +} + +// UnRegister 注销指定key的SQL表达式实现 +// 参数： +// +// key - 要注销的实现标识符 +func UnRegister(key string) { + lock.Lock() + defer lock.Unlock() + if _, ok := ExprMap[key]; ok { + delete(ExprMap, key) + } +} + +// DefaultSQLExpr SQL表达式默认实现 +type DefaultSQLExpr struct { + dimTransform func(string) string + fieldsMap map[string]string +} + +func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { + d.fieldsMap = fieldsMap + return d +} + +// WithTransformDimension 实现设置字段转换方法 +func (d *DefaultSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr { + d.dimTransform = fn + return d +} + +// ParserQueryString 解析查询字符串（当前实现返回空） +func (d *DefaultSQLExpr) ParserQueryString(qs string) (string, error) { + return "", nil +} + +// ParserAllConditions 解析全量条件生成SQL条件表达式 +// 实现逻辑： +// 1. 将多个AND条件组合成OR条件 +// 2. 当有多个OR条件时用括号包裹 +func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + var ( + orConditions []string + ) + + // 遍历所有OR条件组 + for _, conditions := range allConditions { + var andConditions []string + // 处理每个AND条件组 + for _, cond := range conditions { + buildCondition, err := d.buildCondition(cond) + if err != nil { + return "", err + } + if buildCondition != "" { + andConditions = append(andConditions, buildCondition) + } + } + // 合并AND条件 + if len(andConditions) > 0 { + orConditions = append(orConditions, strings.Join(andConditions, " AND ")) + } + } + + // 处理最终OR条件组合 + if len(orConditions) > 0 { + if len(orConditions) == 1 { + return orConditions[0], nil + } + return fmt.Sprintf("(%s)", strings.Join(orConditions, " OR ")), nil + } + + return "", nil +} + +// buildDimensionName 构建维度字段名称（添加反引号包裹） +func (d *DefaultSQLExpr) buildDimensionName(dim string) string { + return fmt.Sprintf("`%s`", dim) +} + +// buildCondition 构建单个条件表达式 +// 参数： +// +// c - 条件字段对象 +// +// 返回值： +// +// 生成的SQL条件表达式字符串和错误信息 +func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { + if len(c.Value) == 0 { + return "", nil + } + + var ( + key string + op string + val string + ) + + if d.dimTransform != nil { + key = d.dimTransform(c.DimensionName) + } else { + key = d.buildDimensionName(c.DimensionName) + } + + // 根据操作符类型生成不同的SQL表达式 + switch c.Operator { + // 处理等于类操作符（=, IN） + case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) == 1 { + if c.IsWildcard { + op = "LIKE" + val = fmt.Sprintf("'%%%s%%'", c.Value[0]) + } else { + op = "=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } + } else { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理不等于类操作符（!=, NOT IN） + case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) == 1 { + op = "!=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } else { + op = "NOT IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理正则表达式匹配 + case metadata.ConditionRegEqual: + op = "REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) // 多个值用|连接 + case metadata.ConditionNotRegEqual: + op = "NOT REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) + // 处理数值比较操作符（>, >=, <, <=） + case metadata.ConditionGt: + op = ">" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionGte: + op = ">=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLt: + op = "<" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLte: + op = "<=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + default: + return "", fmt.Errorf("unknown operator %s", c.Operator) + } + + return fmt.Sprintf("%s %s %s", key, op, val), nil +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go new file mode 100644 index 000000000..46f46b56b --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go @@ -0,0 +1,139 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +// TestParserAllConditions 测试全条件解析的主测试函数 +func TestParserAllConditions(t *testing.T) { + d := &sqlExpr.DefaultSQLExpr{} + + t.Run("空条件测试", func(t *testing.T) { + conditions := metadata.AllConditions{} + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "", result) + }) + + t.Run("单OR组单AND条件", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "host", Operator: metadata.ConditionEqual, Value: []string{"server1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`host` = 'server1'", result) + }) + + t.Run("多OR组组合", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "os", Operator: metadata.ConditionEqual, Value: []string{"linux"}}, + {DimensionName: "status", Operator: metadata.ConditionGt, Value: []string{"5"}}, + }, + { + {DimensionName: "region", Operator: metadata.ConditionEqual, Value: []string{"north", "south"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`os` = 'linux' AND `status` > 5 OR `region` IN ('north', 'south'))" + assert.Equal(t, expected, result) + }) + + t.Run("包含空AND条件过滤", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "log", Operator: metadata.ConditionRegEqual, Value: []string{}}, // 空值条件 + {DimensionName: "level", Operator: metadata.ConditionNotEqual, Value: []string{"debug"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`level` != 'debug'", result) + }) + + t.Run("混合操作符测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "cpu", Operator: metadata.ConditionGte, Value: []string{"80"}}, + {DimensionName: "memory", Operator: metadata.ConditionLt, Value: []string{"90"}}, + }, + { + {DimensionName: "service", Operator: metadata.ConditionContains, Value: []string{"api", "db"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`cpu` >= 80 AND `memory` < 90 OR `service` IN ('api', 'db'))" + assert.Equal(t, expected, result) + }) + + t.Run("错误条件传递测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "error", Operator: metadata.ConditionGt, Value: []string{"1", "2"}}, + }, + } + _, err := d.ParserAllConditions(conditions) + assert.Error(t, err) + assert.Contains(t, err.Error(), "operator > only support 1 value") + }) + + t.Run("多层嵌套组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "A", Operator: metadata.ConditionEqual, Value: []string{"1"}}, + {DimensionName: "B", Operator: metadata.ConditionNotEqual, Value: []string{"2"}}, + }, + { + {DimensionName: "C", Operator: metadata.ConditionRegEqual, Value: []string{"test.*"}}, + }, + { + {DimensionName: "D", Operator: metadata.ConditionLte, Value: []string{"100"}}, + {DimensionName: "E", Operator: metadata.ConditionContains, Value: []string{"x", "y"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + expected := "(`A` = '1' AND `B` != '2' OR `C` REGEXP 'test.*' OR `D` <= 100 AND `E` IN ('x', 'y'))" + assert.Equal(t, expected, result) + }) + + t.Run("单值IN转换测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "version", Operator: metadata.ConditionEqual, Value: []string{"v1"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`version` = 'v1'", result) + }) + + t.Run("正则表达式组合测试", func(t *testing.T) { + conditions := metadata.AllConditions{ + { + {DimensionName: "path", Operator: metadata.ConditionRegEqual, Value: []string{"^/api", "v2$"}}, + }, + } + result, err := d.ParserAllConditions(conditions) + assert.NoError(t, err) + assert.Equal(t, "`path` REGEXP '^/api|v2$'", result) + }) +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go new file mode 100644 index 000000000..42ffbfb0f --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -0,0 +1,170 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr + +import ( + "fmt" + "strings" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" +) + +const ( + Doris = "doris" + DorisTypeText = "text" +) + +type DorisSQLExpr struct { + DefaultSQLExpr + + fieldsMap map[string]string +} + +var _ SQLExpr = (*DorisSQLExpr)(nil) + +func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { + s.fieldsMap = fieldsMap + s.DefaultSQLExpr.WithFieldsMap(fieldsMap).WithTransformDimension(s.fieldFormat) + return s +} + +func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { + expr, err := querystring.Parse(qs) + if err != nil { + return "", err + } + if expr == nil { + return "", nil + } + + return s.walk(expr) +} + +func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + return s.DefaultSQLExpr.ParserAllConditions(allConditions) +} + +func (s *DorisSQLExpr) checkMatchALL(k string) bool { + if s.fieldsMap != nil { + if t, ok := s.fieldsMap[k]; ok { + if t == DorisTypeText { + return true + } + } + } + return false +} + +func (s *DorisSQLExpr) fieldFormat(field string) string { + fs := strings.Split(field, ".") + if len(fs) > 1 { + return fmt.Sprintf(`CAST(%s["%s"] AS STRING)`, fs[0], strings.Join(fs[1:], `"]["`)) + } + return fmt.Sprintf("`%s`", field) +} + +func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { + var ( + err error + left string + right string + ) + + if s == nil { + return "", nil + } + + switch c := e.(type) { + case *querystring.NotExpr: + left, err = s.walk(c.Expr) + if err != nil { + return "", err + } + return fmt.Sprintf("NOT (%s)", left), nil + case *querystring.OrExpr: + left, err = s.walk(c.Left) + if err != nil { + return "", err + } + right, err = s.walk(c.Right) + if err != nil { + return "", err + } + return fmt.Sprintf("(%s OR %s)", left, right), nil + case *querystring.AndExpr: + left, err = s.walk(c.Left) + if err != nil { + return "", err + } + right, err = s.walk(c.Right) + if err != nil { + return "", err + } + return fmt.Sprintf("%s AND %s", left, right), nil + case *querystring.WildcardExpr: + if c.Field == "" { + err = fmt.Errorf(Doris + " " + ErrorMatchAll) + return "", err + } + + return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + case *querystring.MatchExpr: + if c.Field == "" { + err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) + return "", err + } + + if s.checkMatchALL(c.Field) { + return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + } + + return fmt.Sprintf("%s = '%s'", s.fieldFormat(c.Field), c.Value), nil + case *querystring.NumberRangeExpr: + if c.Field == "" { + err = fmt.Errorf(Doris + " " + ErrorMatchAll) + return "", err + } + + var ( + start string + end string + ) + if *c.Start != "*" { + var op string + if c.IncludeStart { + op = ">=" + } else { + op = ">" + } + start = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.Start) + } + + if *c.End != "*" { + var op string + if c.IncludeEnd { + op = "<=" + } else { + op = "<" + } + end = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.End) + } + + return fmt.Sprintf("%s", strings.Join([]string{start, end}, " AND ")), nil + default: + err = fmt.Errorf("expr type is not match %T", e) + } + + return "", err +} + +func init() { + Register(Doris, &DorisSQLExpr{}) +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go new file mode 100644 index 000000000..a00939783 --- /dev/null +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -0,0 +1,240 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package sqlExpr_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" +) + +func TestDorisSQLExpr_ParserQueryString(t *testing.T) { + tests := []struct { + name string + input string + want string + err string + }{ + { + name: "simple match", + input: "name:test", + want: "`name` = 'test'", + }, + { + name: "one word", + input: "test", + err: "doris 不支持全字段检索: test", + }, + { + name: "complex nested query", + input: "(a:1 AND (b:2 OR c:3)) OR NOT d:4", + want: "(`a` = '1' AND (`b` = '2' OR `c` = '3') OR NOT (`d` = '4'))", + }, + { + name: "invalid syntax", + input: "name:test AND OR", + err: "syntax error: unexpected tOR", + }, + { + name: "empty input", + input: "", + }, + { + name: "OR expression with multiple terms", + input: "a:1 OR b:2 OR c:3", + want: "(`a` = '1' OR (`b` = '2' OR `c` = '3'))", + }, + { + name: "mixed AND/OR with proper precedence", + input: "a:1 AND b:2 OR c:3", + want: "`a` = '1' AND (`b` = '2' OR `c` = '3')", + }, + { + name: "exact match with quotes", + input: "name:\"exact match\"", + want: "`name` = 'exact match'", + }, + { + name: "numeric equality", + input: "age:25", + want: "`age` = '25'", + }, + { + name: "date range query", + input: "timestamp:[2023-01-01 TO 2023-12-31]", + want: "( timestamp >= '2023-01-01' AND timestamp <= '2023-12-31' )", + err: "syntax error: unexpected tSTRING, expecting tPHRASE or tNUMBER or tSTAR or tMINUS", + }, + { + name: "invalid field name", + input: "123field:value", + want: "`123field` = 'value'", + }, + { + name: "text filter", + input: "text:value", + want: "`text` LIKE '%value%'", + }, + { + name: "object field", + input: "__ext.container_name: value", + want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "text": sqlExpr.DorisTypeText, + }).ParserQueryString(tt.input) + if err != nil { + assert.Equal(t, err.Error(), tt.err) + return + } + + assert.Equal(t, tt.want, got) + }) + } +} + +// TestDorisSQLExpr_ParserAllConditions 单元测试 +func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { + tests := []struct { + name string + condition metadata.AllConditions + want string + wantErr error + }{ + { + name: "test object field condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"test-value"}, + Operator: metadata.ConditionEqual, + }, + { + DimensionName: "tag", + Value: []string{"test"}, + Operator: metadata.ConditionNotEqual, + }, + }, + }, + want: "CAST(object[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", + }, + { + name: "test text field wildcard", + condition: metadata.AllConditions{ + { + { + DimensionName: "object.field", + Value: []string{"partial"}, + Operator: metadata.ConditionContains, + IsWildcard: true, + }, + }, + }, + want: `CAST(object["field"] AS STRING) LIKE '%partial%'`, + }, + { + name: "test OR condition", + condition: metadata.AllConditions{ + { + { + DimensionName: "status", + Value: []string{"running"}, + Operator: metadata.ConditionEqual, + }, + }, + { + { + DimensionName: "code", + Value: []string{"500"}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "(`status` = 'running' OR `code` = '500')", + }, + { + name: "test numeric field without cast", + condition: metadata.AllConditions{ + { + { + DimensionName: "cpu_usage", + Value: []string{"80"}, + Operator: metadata.ConditionGt, + }, + }, + }, + want: "`cpu_usage` > 80", + }, + { + name: "test IN operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "env", + Value: []string{"prod", "test"}, + Operator: metadata.ConditionContains, + }, + }, + }, + want: "`env` IN ('prod', 'test')", + }, + { + name: "test empty value", + condition: metadata.AllConditions{ + { + { + DimensionName: "host", + Value: []string{}, + Operator: metadata.ConditionEqual, + }, + }, + }, + want: "", + }, + { + name: "test invalid operator", + condition: metadata.AllConditions{ + { + { + DimensionName: "time", + Value: []string{"2023"}, + Operator: "unknown", + }, + }, + }, + wantErr: fmt.Errorf("unknown operator unknown"), + }, + } + + e := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + "object.field": sqlExpr.DorisTypeText, + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := e.ParserAllConditions(tt.condition) + + if err != nil { + assert.Equal(t, tt.wantErr, err) + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 80b28b4a2..69da463ad 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -314,10 +314,21 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac ) if errors.As(err, &e) { if e.Details != nil { - for _, rc := range e.Details.RootCause { - msg.WriteString(fmt.Sprintf("%s: %s, ", rc.Index, rc.Reason)) + if len(e.Details.RootCause) > 0 { + msg.WriteString("root cause: \n") + for _, rc := range e.Details.RootCause { + msg.WriteString(fmt.Sprintf("%s: %s \n", rc.Index, rc.Reason)) + } + } + + if e.Details.CausedBy != nil { + msg.WriteString("caused by: \n") + for k, v := range e.Details.CausedBy { + msg.WriteString(fmt.Sprintf("%s: %v \n", k, v)) + } } } + return nil, errors.New(msg.String()) } else { return nil, err @@ -626,7 +637,7 @@ func (i *Instance) QuerySeriesSet( err error ) - ctx, span := trace.NewSpan(ctx, "elasticsearch-query-reference") + ctx, span := trace.NewSpan(ctx, "elasticsearch-query-series-set") defer span.End(&err) if len(query.Aggregates) == 0 { From 430111351a92886cbdec38bd25920bcbd80d6766 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 23:35:55 +0800 Subject: [PATCH 022/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/metadata/struct.go | 3 +- pkg/unify-query/mock/handler.go | 2 +- pkg/unify-query/query/structured/method.go | 2 + pkg/unify-query/query/structured/query_ts.go | 1 + pkg/unify-query/tsdb/bksql/format.go | 100 +++------ pkg/unify-query/tsdb/bksql/instance.go | 34 +++- pkg/unify-query/tsdb/bksql/instance_test.go | 204 +++++++++++++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 86 ++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 19 +- 9 files changed, 329 insertions(+), 122 deletions(-) diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 1081d8eec..18db5c09c 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -58,7 +58,8 @@ type TimeField struct { // Aggregate 聚合方法 type Aggregate struct { - Name string + Name string + Field string Dimensions []string Without bool diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index d6edd5a47..290f19b27 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -211,7 +211,7 @@ func mockBkSQLHandler(ctx context.Context) { d, ok := BkSQL.Get(request.Sql) if !ok { - err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) + err = fmt.Errorf(`bksql mock data is empty in: %s`, request.Sql) return } diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index ef0a3aada..93aa652ff 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -68,6 +68,8 @@ func (a AggregateMethodList) ToQry(timezone string) metadata.Aggregates { type AggregateMethod struct { // Method 聚合方法 Method string `json:"method,omitempty" example:"mean"` + // Field 聚合字段，默认为指标字段，指定则会进行覆盖 + Field string `json:"field,omitempty" example:"field"` // Without Without bool `json:"without" example:false` // Dimensions 聚合维度 diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 8df8cc057..7300c553e 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -469,6 +469,7 @@ func (q *Query) Aggregates() (aggs metadata.Aggregates, err error) { if name, ok := domSampledFunc[am.Method+q.TimeAggregation.Function]; ok { agg := metadata.Aggregate{ Name: name, + Field: am.Field, Dimensions: am.Dimensions, Without: am.Without, Window: time.Duration(window), diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 9cb910057..2d0ed41ea 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -25,6 +25,8 @@ import ( ) const ( + selectAll = "*" + dtEventTimeStamp = "dtEventTimeStamp" dtEventTime = "dtEventTime" localTime = "localTime" @@ -62,22 +64,18 @@ type QueryFactory struct { end time.Time step time.Duration - selects []string - groups []string - orders metadata.Orders - - sql strings.Builder + orders metadata.Orders timeField string + + expr sqlExpr.SQLExpr } func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f := &QueryFactory{ - ctx: ctx, - query: query, - selects: make([]string, 0), - groups: make([]string, 0), - orders: make(metadata.Orders), + ctx: ctx, + query: query, + orders: make(metadata.Orders), } if query.Orders != nil { for k, v := range query.Orders { @@ -90,11 +88,10 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { } else { f.timeField = dtEventTimeStamp } - return f -} -func (f *QueryFactory) write(s string) { - f.sql.WriteString(s + " ") + fieldsMap := make(map[string]string) + f.expr = sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap).WithInternalFields(f.timeField, query.Field) + return f } func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { @@ -103,36 +100,6 @@ func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { return f } -func (f *QueryFactory) ParserQuery() (err error) { - f.lock.Lock() - defer f.lock.Unlock() - - if len(f.query.Aggregates) > 0 { - for _, agg := range f.query.Aggregates { - for _, dim := range agg.Dimensions { - dim = fmt.Sprintf("`%s`", dim) - f.groups = append(f.groups, dim) - f.selects = append(f.selects, dim) - } - f.selects = append(f.selects, fmt.Sprintf("%s(`%s`) AS `%s`", strings.ToUpper(agg.Name), f.query.Field, value)) - if agg.Window > 0 { - timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", f.timeField, f.timeField, agg.Window.Milliseconds()) - f.groups = append(f.groups, timeField) - f.selects = append(f.selects, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) - f.orders[FieldTime] = true - } - } - } - - if len(f.selects) == 0 { - f.selects = append(f.selects, "*") - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.query.Field, value)) - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.timeField, timeStamp)) - } - - return -} - func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") @@ -170,12 +137,9 @@ func (f *QueryFactory) BuildWhere() (string, error) { s = append(s, theDateFilter) } - fieldsMap := make(map[string]string) - expr := sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap) - // QueryString to sql if f.query.QueryString != "" { - qs, err := expr.ParserQueryString(f.query.QueryString) + qs, err := f.expr.ParserQueryString(f.query.QueryString) if err != nil { return "", err } @@ -187,7 +151,7 @@ func (f *QueryFactory) BuildWhere() (string, error) { // AllConditions to sql if len(f.query.AllConditions) > 0 { - qs, err := expr.ParserAllConditions(f.query.AllConditions) + qs, err := f.expr.ParserAllConditions(f.query.AllConditions) if err != nil { return "", err } @@ -201,36 +165,26 @@ func (f *QueryFactory) BuildWhere() (string, error) { } func (f *QueryFactory) SQL() (sql string, err error) { - f.sql.Reset() - err = f.ParserQuery() + selectString, groupString, err := f.expr.ParserAggregates(f.query.Aggregates) if err != nil { return } - f.lock.RLock() - defer f.lock.RUnlock() - - f.write("SELECT") - f.write(strings.Join(f.selects, ", ")) - f.write("FROM") - db := fmt.Sprintf("`%s`", f.query.DB) + table := fmt.Sprintf("`%s`", f.query.DB) if f.query.Measurement != "" { - db += "." + f.query.Measurement + table += "." + f.query.Measurement } - f.write(db) - where, err := f.BuildWhere() + sql += fmt.Sprintf("SELECT %s FROM %s", selectString, table) + whereString, err := f.BuildWhere() if err != nil { return } - if where != "" { - f.write("WHERE") - f.write(where) + if whereString != "" { + sql += " WHERE " + whereString } - - if len(f.groups) > 0 { - f.write("GROUP BY") - f.write(strings.Join(f.groups, ", ")) + if groupString != "" { + sql += " GROUP BY " + groupString } orders := make([]string, 0) @@ -252,19 +206,15 @@ func (f *QueryFactory) SQL() (sql string, err error) { } if len(orders) > 0 { sort.Strings(orders) - f.write("ORDER BY") - f.write(strings.Join(orders, ", ")) + sql += " ORDER BY " + strings.Join(orders, ", ") } if f.query.From > 0 { - f.write("OFFSET") - f.write(fmt.Sprintf("%d", f.query.From)) + sql += fmt.Sprintf(" OFFSET %d", f.query.From) } if f.query.Size > 0 { - f.write("LIMIT") - f.write(fmt.Sprintf("%d", f.query.Size)) + sql += fmt.Sprintf(" LIMIT %d", f.query.Size) } - sql = strings.Trim(f.sql.String(), " ") return } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 91867371a..59641356e 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -403,14 +403,17 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } func (i *Instance) DirectQueryRange(ctx context.Context, promql string, start, end time.Time, step time.Duration) (promql.Matrix, error) { + log.Warnf(ctx, "%s not support direct query range", i.InstanceType()) return nil, nil } func (i *Instance) DirectQuery(ctx context.Context, qs string, end time.Time) (promql.Vector, error) { + log.Warnf(ctx, "%s not support direct query", i.InstanceType()) return nil, nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { + log.Warnf(ctx, "%s not support query exemplar", i.InstanceType()) return nil, nil } @@ -422,12 +425,15 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s ctx, span := trace.NewSpan(ctx, "bk-sql-label-name") defer span.End(&err) - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 取字段名不需要返回数据，但是 size 不能使用 0，所以还是用 1 + query.Size = 1 + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + sql, err := queryFactory.SQL() + if err != nil { + return nil, err } - sql := fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", query.Measurement, where) + data, err := i.sqlQuery(ctx, sql, span) if err != nil { return nil, err @@ -451,12 +457,20 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, return nil, fmt.Errorf("not support metric query with %s", name) } - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 使用聚合的方式统计维度组合 + query.Aggregates = metadata.Aggregates{ + { + Dimensions: []string{name}, + Name: "count", + }, } - sql := fmt.Sprintf("SELECT COUNT(`%s`) AS `%s`, %s FROM %s WHERE %s GROUP BY %s", query.Field, query.Field, name, query.Measurement, where, name) + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + sql, err := queryFactory.SQL() + if err != nil { + return nil, err + } + data, err := i.sqlQuery(ctx, sql, span) if err != nil { return nil, err diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index e169bdcfc..7a4d8d894 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -27,20 +27,8 @@ import ( func TestInstance_QueryRaw(t *testing.T) { - ctx := context.Background() - mock.Init() - - ins, err := bksql.NewInstance(ctx, &bksql.Options{ - Address: mock.BkSQLUrl, - Timeout: time.Minute, - MaxLimit: 1e4, - Tolerance: 5, - Curl: &curl.HttpCurl{Log: log.DefaultLogger}, - }) - if err != nil { - log.Fatalf(ctx, err.Error()) - return - } + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", @@ -401,6 +389,71 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", }, + + // 测试用例8: 默认处理 object 字段 + { + name: "default multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "__ext.container_id", + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.container_id", + Operator: metadata.ConditionEqual, + Value: []string{"1234567890"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.container_id", "test"}, + }, + }, + Orders: map[string]bool{ + "timestamp": true, + "__ext.container_id": false, + }, + }, + expected: "SELECT `__ext.container_id`, `test`, COUNT(`__ext.container_id`) AS `_value_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `__ext.container_id` = '1234567890' GROUP BY `__ext.container_id`, `test` ORDER BY `__ext.container_id` DESC, `timestamp` ASC", + }, + + // 测试用例8: doris 处理 object 字段 + { + name: "default multiple order fields", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.container_id", + Operator: metadata.ConditionEqual, + Value: []string{"5782c24d05ae0eceb3074d6ae492e7b027b7310711a4b63392eacd9314fc9a83"}, + }, + { + DimensionName: "bk_host_id", + Operator: metadata.ConditionEqual, + Value: []string{"267730"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.container_id", "bk_host_id"}, + }, + }, + Orders: map[string]bool{ + "timestamp": true, + "__ext.container_id": false, + }, + }, + expected: "", + }, } for _, tc := range testCases { @@ -443,3 +496,126 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }) } } + +// 测试正常标签名查询 +func TestInstance_QueryLabelNames_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + }, + expectedNames: []string{ + "dteventtimestamp", "dteventtime", "localtime", "_starttime_", "_endtime_", "bk_host_id", "__ext", "cloudid", "path", "gseindex", "iterationindex", "log", "time", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelNames(ctx, tt.qry, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 测试正常标签名查询 +func TestInstance_QueryLabelValues_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + key string + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Size: 2, + }, + key: "bk_host_id", + expectedNames: []string{ + "5843771", "4580470", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelValues(ctx, tt.qry, tt.key, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 创建测试用Instance +func createTestInstance(ctx context.Context) *bksql.Instance { + mock.Init() + + ins, err := bksql.NewInstance(ctx, &bksql.Options{ + Address: mock.BkSQLUrl, + Timeout: time.Minute, + MaxLimit: 1e4, + Tolerance: 5, + Curl: &curl.HttpCurl{Log: log.DefaultLogger}, + }) + if err != nil { + log.Fatalf(ctx, err.Error()) + return nil + } + return ins +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index fc8a9bdc7..adf06f874 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -13,10 +13,19 @@ import ( "fmt" "strings" "sync" + "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" ) +const ( + selectAll = "*" + timeStamp = "_timestamp_" + value = "_value_" + + theDate = "thedate" +) + // ErrorMatchAll 定义全字段检索错误提示信息 var ( ErrorMatchAll = "不支持全字段检索" @@ -29,10 +38,14 @@ type SQLExpr interface { WithFieldsMap(fieldsMap map[string]string) SQLExpr // WithTransformDimension 设置字段转换方法 WithTransformDimension(func(string) string) SQLExpr + // WithInternalFields 设置内部字段 + WithInternalFields(timeField, valueField string) SQLExpr // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 ParserQueryString(qs string) (string, error) // ParserAllConditions 解析全量条件生成SQL条件表达式 ParserAllConditions(allConditions metadata.AllConditions) (string, error) + // ParserAggregates 解析聚合条件生成SQL条件表达式 + ParserAggregates(aggregates metadata.Aggregates) (string, string, error) } // SQL表达式注册管理相关变量 @@ -57,7 +70,13 @@ func GetSQLExpr(key string) SQLExpr { if sqlExpr, ok := ExprMap[key]; ok { return sqlExpr } else { - return &DefaultSQLExpr{} + return (&DefaultSQLExpr{}).WithTransformDimension(func(s string) string { + if s == "" { + return "" + } + + return fmt.Sprintf("`%s`", s) + }) } } @@ -88,6 +107,18 @@ func UnRegister(key string) { type DefaultSQLExpr struct { dimTransform func(string) string fieldsMap map[string]string + + timeField string + valueField string + + start time.Time + end time.Time +} + +func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { + d.timeField = timeField + d.valueField = valueField + return d } func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { @@ -102,10 +133,50 @@ func (d *DefaultSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr } // ParserQueryString 解析查询字符串（当前实现返回空） -func (d *DefaultSQLExpr) ParserQueryString(qs string) (string, error) { +func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { return "", nil } +// ParserAggregates 解析聚合函数，生成 select 和 group by 字段 +func (d *DefaultSQLExpr) ParserAggregates(aggregates metadata.Aggregates) (string, string, error) { + var ( + selectFields []string + groupByFields []string + ) + valueField := d.dimTransform(d.valueField) + + for _, agg := range aggregates { + for _, dim := range agg.Dimensions { + dim = d.dimTransform(dim) + selectFields = append(selectFields, dim) + groupByFields = append(groupByFields, dim) + } + + if valueField == "" { + valueField = selectAll + } + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, value)) + + if agg.Window > 0 { + timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", d.timeField, d.timeField, agg.Window.Milliseconds()) + groupByFields = append(groupByFields, timeField) + selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) + } + } + + if len(selectFields) == 0 { + selectFields = append(selectFields, selectAll) + if valueField != "" { + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, value)) + } + if d.timeField != "" { + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, timeStamp)) + } + } + + return strings.Join(selectFields, ", "), strings.Join(groupByFields, ", "), nil +} + // ParserAllConditions 解析全量条件生成SQL条件表达式 // 实现逻辑： // 1. 将多个AND条件组合成OR条件 @@ -145,11 +216,6 @@ func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllCondition return "", nil } -// buildDimensionName 构建维度字段名称（添加反引号包裹） -func (d *DefaultSQLExpr) buildDimensionName(dim string) string { - return fmt.Sprintf("`%s`", dim) -} - // buildCondition 构建单个条件表达式 // 参数： // @@ -169,11 +235,7 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro val string ) - if d.dimTransform != nil { - key = d.dimTransform(c.DimensionName) - } else { - key = d.buildDimensionName(c.DimensionName) - } + key = d.dimTransform(c.DimensionName) // 根据操作符类型生成不同的SQL表达式 switch c.Operator { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 42ffbfb0f..b12702812 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -32,7 +32,8 @@ var _ SQLExpr = (*DorisSQLExpr)(nil) func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { s.fieldsMap = fieldsMap - s.DefaultSQLExpr.WithFieldsMap(fieldsMap).WithTransformDimension(s.fieldFormat) + + s.DefaultSQLExpr.WithFieldsMap(fieldsMap) return s } @@ -49,7 +50,7 @@ func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { } func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { - return s.DefaultSQLExpr.ParserAllConditions(allConditions) + return s.DefaultSQLExpr.WithTransformDimension(s.dimTransform).ParserAllConditions(allConditions) } func (s *DorisSQLExpr) checkMatchALL(k string) bool { @@ -63,10 +64,10 @@ func (s *DorisSQLExpr) checkMatchALL(k string) bool { return false } -func (s *DorisSQLExpr) fieldFormat(field string) string { +func (s *DorisSQLExpr) dimTransform(field string) string { fs := strings.Split(field, ".") if len(fs) > 1 { - return fmt.Sprintf(`CAST(%s["%s"] AS STRING)`, fs[0], strings.Join(fs[1:], `"]["`)) + return fmt.Sprintf("CAST(`%s`[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)) } return fmt.Sprintf("`%s`", field) } @@ -115,7 +116,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } - return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil case *querystring.MatchExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) @@ -123,10 +124,10 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } if s.checkMatchALL(c.Field) { - return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil } - return fmt.Sprintf("%s = '%s'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s = '%s'", s.dimTransform(c.Field), c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll) @@ -144,7 +145,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = ">" } - start = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.Start) + start = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.Start) } if *c.End != "*" { @@ -154,7 +155,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = "<" } - end = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.End) + end = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.End) } return fmt.Sprintf("%s", strings.Join([]string{start, end}, " AND ")), nil From 25c60064d28aa74daea04ec5ce4189875ceed7de Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Feb 2025 23:35:55 +0800 Subject: [PATCH 023/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504=20#=20R?= =?UTF-8?q?eviewed,=20transaction=20id:=2031689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/metadata/struct.go | 3 +- pkg/unify-query/mock/handler.go | 2 +- pkg/unify-query/query/structured/method.go | 2 + pkg/unify-query/query/structured/query_ts.go | 1 + pkg/unify-query/tsdb/bksql/format.go | 100 +++------ pkg/unify-query/tsdb/bksql/instance.go | 34 +++- pkg/unify-query/tsdb/bksql/instance_test.go | 204 +++++++++++++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 86 ++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 19 +- 9 files changed, 329 insertions(+), 122 deletions(-) diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 1081d8eec..18db5c09c 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -58,7 +58,8 @@ type TimeField struct { // Aggregate 聚合方法 type Aggregate struct { - Name string + Name string + Field string Dimensions []string Without bool diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index d6edd5a47..290f19b27 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -211,7 +211,7 @@ func mockBkSQLHandler(ctx context.Context) { d, ok := BkSQL.Get(request.Sql) if !ok { - err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) + err = fmt.Errorf(`bksql mock data is empty in: %s`, request.Sql) return } diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index ef0a3aada..93aa652ff 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -68,6 +68,8 @@ func (a AggregateMethodList) ToQry(timezone string) metadata.Aggregates { type AggregateMethod struct { // Method 聚合方法 Method string `json:"method,omitempty" example:"mean"` + // Field 聚合字段，默认为指标字段，指定则会进行覆盖 + Field string `json:"field,omitempty" example:"field"` // Without Without bool `json:"without" example:false` // Dimensions 聚合维度 diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 8df8cc057..7300c553e 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -469,6 +469,7 @@ func (q *Query) Aggregates() (aggs metadata.Aggregates, err error) { if name, ok := domSampledFunc[am.Method+q.TimeAggregation.Function]; ok { agg := metadata.Aggregate{ Name: name, + Field: am.Field, Dimensions: am.Dimensions, Without: am.Without, Window: time.Duration(window), diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 9cb910057..2d0ed41ea 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -25,6 +25,8 @@ import ( ) const ( + selectAll = "*" + dtEventTimeStamp = "dtEventTimeStamp" dtEventTime = "dtEventTime" localTime = "localTime" @@ -62,22 +64,18 @@ type QueryFactory struct { end time.Time step time.Duration - selects []string - groups []string - orders metadata.Orders - - sql strings.Builder + orders metadata.Orders timeField string + + expr sqlExpr.SQLExpr } func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f := &QueryFactory{ - ctx: ctx, - query: query, - selects: make([]string, 0), - groups: make([]string, 0), - orders: make(metadata.Orders), + ctx: ctx, + query: query, + orders: make(metadata.Orders), } if query.Orders != nil { for k, v := range query.Orders { @@ -90,11 +88,10 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { } else { f.timeField = dtEventTimeStamp } - return f -} -func (f *QueryFactory) write(s string) { - f.sql.WriteString(s + " ") + fieldsMap := make(map[string]string) + f.expr = sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap).WithInternalFields(f.timeField, query.Field) + return f } func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { @@ -103,36 +100,6 @@ func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { return f } -func (f *QueryFactory) ParserQuery() (err error) { - f.lock.Lock() - defer f.lock.Unlock() - - if len(f.query.Aggregates) > 0 { - for _, agg := range f.query.Aggregates { - for _, dim := range agg.Dimensions { - dim = fmt.Sprintf("`%s`", dim) - f.groups = append(f.groups, dim) - f.selects = append(f.selects, dim) - } - f.selects = append(f.selects, fmt.Sprintf("%s(`%s`) AS `%s`", strings.ToUpper(agg.Name), f.query.Field, value)) - if agg.Window > 0 { - timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", f.timeField, f.timeField, agg.Window.Milliseconds()) - f.groups = append(f.groups, timeField) - f.selects = append(f.selects, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) - f.orders[FieldTime] = true - } - } - } - - if len(f.selects) == 0 { - f.selects = append(f.selects, "*") - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.query.Field, value)) - f.selects = append(f.selects, fmt.Sprintf("`%s` AS `%s`", f.timeField, timeStamp)) - } - - return -} - func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") @@ -170,12 +137,9 @@ func (f *QueryFactory) BuildWhere() (string, error) { s = append(s, theDateFilter) } - fieldsMap := make(map[string]string) - expr := sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap) - // QueryString to sql if f.query.QueryString != "" { - qs, err := expr.ParserQueryString(f.query.QueryString) + qs, err := f.expr.ParserQueryString(f.query.QueryString) if err != nil { return "", err } @@ -187,7 +151,7 @@ func (f *QueryFactory) BuildWhere() (string, error) { // AllConditions to sql if len(f.query.AllConditions) > 0 { - qs, err := expr.ParserAllConditions(f.query.AllConditions) + qs, err := f.expr.ParserAllConditions(f.query.AllConditions) if err != nil { return "", err } @@ -201,36 +165,26 @@ func (f *QueryFactory) BuildWhere() (string, error) { } func (f *QueryFactory) SQL() (sql string, err error) { - f.sql.Reset() - err = f.ParserQuery() + selectString, groupString, err := f.expr.ParserAggregates(f.query.Aggregates) if err != nil { return } - f.lock.RLock() - defer f.lock.RUnlock() - - f.write("SELECT") - f.write(strings.Join(f.selects, ", ")) - f.write("FROM") - db := fmt.Sprintf("`%s`", f.query.DB) + table := fmt.Sprintf("`%s`", f.query.DB) if f.query.Measurement != "" { - db += "." + f.query.Measurement + table += "." + f.query.Measurement } - f.write(db) - where, err := f.BuildWhere() + sql += fmt.Sprintf("SELECT %s FROM %s", selectString, table) + whereString, err := f.BuildWhere() if err != nil { return } - if where != "" { - f.write("WHERE") - f.write(where) + if whereString != "" { + sql += " WHERE " + whereString } - - if len(f.groups) > 0 { - f.write("GROUP BY") - f.write(strings.Join(f.groups, ", ")) + if groupString != "" { + sql += " GROUP BY " + groupString } orders := make([]string, 0) @@ -252,19 +206,15 @@ func (f *QueryFactory) SQL() (sql string, err error) { } if len(orders) > 0 { sort.Strings(orders) - f.write("ORDER BY") - f.write(strings.Join(orders, ", ")) + sql += " ORDER BY " + strings.Join(orders, ", ") } if f.query.From > 0 { - f.write("OFFSET") - f.write(fmt.Sprintf("%d", f.query.From)) + sql += fmt.Sprintf(" OFFSET %d", f.query.From) } if f.query.Size > 0 { - f.write("LIMIT") - f.write(fmt.Sprintf("%d", f.query.Size)) + sql += fmt.Sprintf(" LIMIT %d", f.query.Size) } - sql = strings.Trim(f.sql.String(), " ") return } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 91867371a..59641356e 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -403,14 +403,17 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } func (i *Instance) DirectQueryRange(ctx context.Context, promql string, start, end time.Time, step time.Duration) (promql.Matrix, error) { + log.Warnf(ctx, "%s not support direct query range", i.InstanceType()) return nil, nil } func (i *Instance) DirectQuery(ctx context.Context, qs string, end time.Time) (promql.Vector, error) { + log.Warnf(ctx, "%s not support direct query", i.InstanceType()) return nil, nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { + log.Warnf(ctx, "%s not support query exemplar", i.InstanceType()) return nil, nil } @@ -422,12 +425,15 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s ctx, span := trace.NewSpan(ctx, "bk-sql-label-name") defer span.End(&err) - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 取字段名不需要返回数据，但是 size 不能使用 0，所以还是用 1 + query.Size = 1 + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + sql, err := queryFactory.SQL() + if err != nil { + return nil, err } - sql := fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", query.Measurement, where) + data, err := i.sqlQuery(ctx, sql, span) if err != nil { return nil, err @@ -451,12 +457,20 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, return nil, fmt.Errorf("not support metric query with %s", name) } - where := fmt.Sprintf("%s >= %d AND %s < %d", dtEventTimeStamp, start.UnixMilli(), dtEventTimeStamp, end.UnixMilli()) - // 拼接过滤条件 - if query.BkSqlCondition != "" { - where = fmt.Sprintf("%s AND (%s)", where, query.BkSqlCondition) + // 使用聚合的方式统计维度组合 + query.Aggregates = metadata.Aggregates{ + { + Dimensions: []string{name}, + Name: "count", + }, } - sql := fmt.Sprintf("SELECT COUNT(`%s`) AS `%s`, %s FROM %s WHERE %s GROUP BY %s", query.Field, query.Field, name, query.Measurement, where, name) + + queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + sql, err := queryFactory.SQL() + if err != nil { + return nil, err + } + data, err := i.sqlQuery(ctx, sql, span) if err != nil { return nil, err diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index e169bdcfc..7a4d8d894 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -27,20 +27,8 @@ import ( func TestInstance_QueryRaw(t *testing.T) { - ctx := context.Background() - mock.Init() - - ins, err := bksql.NewInstance(ctx, &bksql.Options{ - Address: mock.BkSQLUrl, - Timeout: time.Minute, - MaxLimit: 1e4, - Tolerance: 5, - Curl: &curl.HttpCurl{Log: log.DefaultLogger}, - }) - if err != nil { - log.Fatalf(ctx, err.Error()) - return - } + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", @@ -401,6 +389,71 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", }, + + // 测试用例8: 默认处理 object 字段 + { + name: "default multiple order fields", + query: &metadata.Query{ + DB: "transaction_logs", + Field: "__ext.container_id", + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.container_id", + Operator: metadata.ConditionEqual, + Value: []string{"1234567890"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.container_id", "test"}, + }, + }, + Orders: map[string]bool{ + "timestamp": true, + "__ext.container_id": false, + }, + }, + expected: "SELECT `__ext.container_id`, `test`, COUNT(`__ext.container_id`) AS `_value_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `__ext.container_id` = '1234567890' GROUP BY `__ext.container_id`, `test` ORDER BY `__ext.container_id` DESC, `timestamp` ASC", + }, + + // 测试用例8: doris 处理 object 字段 + { + name: "default multiple order fields", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.container_id", + Operator: metadata.ConditionEqual, + Value: []string{"5782c24d05ae0eceb3074d6ae492e7b027b7310711a4b63392eacd9314fc9a83"}, + }, + { + DimensionName: "bk_host_id", + Operator: metadata.ConditionEqual, + Value: []string{"267730"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.container_id", "bk_host_id"}, + }, + }, + Orders: map[string]bool{ + "timestamp": true, + "__ext.container_id": false, + }, + }, + expected: "", + }, } for _, tc := range testCases { @@ -443,3 +496,126 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }) } } + +// 测试正常标签名查询 +func TestInstance_QueryLabelNames_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + }, + expectedNames: []string{ + "dteventtimestamp", "dteventtime", "localtime", "_starttime_", "_endtime_", "bk_host_id", "__ext", "cloudid", "path", "gseindex", "iterationindex", "log", "time", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelNames(ctx, tt.qry, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 测试正常标签名查询 +func TestInstance_QueryLabelValues_Normal(t *testing.T) { + // 初始化测试实例 + ctx := metadata.InitHashID(context.Background()) + instance := createTestInstance(ctx) + + end := time.Unix(1740553771, 0) + start := time.Unix(1740551971, 0) + + // mock 查询数据 + mock.BkSQL.Set(map[string]any{ + "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, + }) + + // 测试用例 + tests := []struct { + name string + qry *metadata.Query + key string + + expectedNames []string + expectError bool + }{ + { + name: "normal-case", + qry: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Size: 2, + }, + key: "bk_host_id", + expectedNames: []string{ + "5843771", "4580470", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 执行测试 + ctx = metadata.InitHashID(ctx) + names, err := instance.QueryLabelValues(ctx, tt.qry, tt.key, start, end) + + // 验证结果 + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expectedNames, names) + } + }) + } +} + +// 创建测试用Instance +func createTestInstance(ctx context.Context) *bksql.Instance { + mock.Init() + + ins, err := bksql.NewInstance(ctx, &bksql.Options{ + Address: mock.BkSQLUrl, + Timeout: time.Minute, + MaxLimit: 1e4, + Tolerance: 5, + Curl: &curl.HttpCurl{Log: log.DefaultLogger}, + }) + if err != nil { + log.Fatalf(ctx, err.Error()) + return nil + } + return ins +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index fc8a9bdc7..adf06f874 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -13,10 +13,19 @@ import ( "fmt" "strings" "sync" + "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" ) +const ( + selectAll = "*" + timeStamp = "_timestamp_" + value = "_value_" + + theDate = "thedate" +) + // ErrorMatchAll 定义全字段检索错误提示信息 var ( ErrorMatchAll = "不支持全字段检索" @@ -29,10 +38,14 @@ type SQLExpr interface { WithFieldsMap(fieldsMap map[string]string) SQLExpr // WithTransformDimension 设置字段转换方法 WithTransformDimension(func(string) string) SQLExpr + // WithInternalFields 设置内部字段 + WithInternalFields(timeField, valueField string) SQLExpr // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 ParserQueryString(qs string) (string, error) // ParserAllConditions 解析全量条件生成SQL条件表达式 ParserAllConditions(allConditions metadata.AllConditions) (string, error) + // ParserAggregates 解析聚合条件生成SQL条件表达式 + ParserAggregates(aggregates metadata.Aggregates) (string, string, error) } // SQL表达式注册管理相关变量 @@ -57,7 +70,13 @@ func GetSQLExpr(key string) SQLExpr { if sqlExpr, ok := ExprMap[key]; ok { return sqlExpr } else { - return &DefaultSQLExpr{} + return (&DefaultSQLExpr{}).WithTransformDimension(func(s string) string { + if s == "" { + return "" + } + + return fmt.Sprintf("`%s`", s) + }) } } @@ -88,6 +107,18 @@ func UnRegister(key string) { type DefaultSQLExpr struct { dimTransform func(string) string fieldsMap map[string]string + + timeField string + valueField string + + start time.Time + end time.Time +} + +func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { + d.timeField = timeField + d.valueField = valueField + return d } func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { @@ -102,10 +133,50 @@ func (d *DefaultSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr } // ParserQueryString 解析查询字符串（当前实现返回空） -func (d *DefaultSQLExpr) ParserQueryString(qs string) (string, error) { +func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { return "", nil } +// ParserAggregates 解析聚合函数，生成 select 和 group by 字段 +func (d *DefaultSQLExpr) ParserAggregates(aggregates metadata.Aggregates) (string, string, error) { + var ( + selectFields []string + groupByFields []string + ) + valueField := d.dimTransform(d.valueField) + + for _, agg := range aggregates { + for _, dim := range agg.Dimensions { + dim = d.dimTransform(dim) + selectFields = append(selectFields, dim) + groupByFields = append(groupByFields, dim) + } + + if valueField == "" { + valueField = selectAll + } + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, value)) + + if agg.Window > 0 { + timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", d.timeField, d.timeField, agg.Window.Milliseconds()) + groupByFields = append(groupByFields, timeField) + selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) + } + } + + if len(selectFields) == 0 { + selectFields = append(selectFields, selectAll) + if valueField != "" { + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, value)) + } + if d.timeField != "" { + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, timeStamp)) + } + } + + return strings.Join(selectFields, ", "), strings.Join(groupByFields, ", "), nil +} + // ParserAllConditions 解析全量条件生成SQL条件表达式 // 实现逻辑： // 1. 将多个AND条件组合成OR条件 @@ -145,11 +216,6 @@ func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllCondition return "", nil } -// buildDimensionName 构建维度字段名称（添加反引号包裹） -func (d *DefaultSQLExpr) buildDimensionName(dim string) string { - return fmt.Sprintf("`%s`", dim) -} - // buildCondition 构建单个条件表达式 // 参数： // @@ -169,11 +235,7 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro val string ) - if d.dimTransform != nil { - key = d.dimTransform(c.DimensionName) - } else { - key = d.buildDimensionName(c.DimensionName) - } + key = d.dimTransform(c.DimensionName) // 根据操作符类型生成不同的SQL表达式 switch c.Operator { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 42ffbfb0f..b12702812 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -32,7 +32,8 @@ var _ SQLExpr = (*DorisSQLExpr)(nil) func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { s.fieldsMap = fieldsMap - s.DefaultSQLExpr.WithFieldsMap(fieldsMap).WithTransformDimension(s.fieldFormat) + + s.DefaultSQLExpr.WithFieldsMap(fieldsMap) return s } @@ -49,7 +50,7 @@ func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { } func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { - return s.DefaultSQLExpr.ParserAllConditions(allConditions) + return s.DefaultSQLExpr.WithTransformDimension(s.dimTransform).ParserAllConditions(allConditions) } func (s *DorisSQLExpr) checkMatchALL(k string) bool { @@ -63,10 +64,10 @@ func (s *DorisSQLExpr) checkMatchALL(k string) bool { return false } -func (s *DorisSQLExpr) fieldFormat(field string) string { +func (s *DorisSQLExpr) dimTransform(field string) string { fs := strings.Split(field, ".") if len(fs) > 1 { - return fmt.Sprintf(`CAST(%s["%s"] AS STRING)`, fs[0], strings.Join(fs[1:], `"]["`)) + return fmt.Sprintf("CAST(`%s`[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)) } return fmt.Sprintf("`%s`", field) } @@ -115,7 +116,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } - return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil case *querystring.MatchExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) @@ -123,10 +124,10 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } if s.checkMatchALL(c.Field) { - return fmt.Sprintf("%s LIKE '%%%s%%'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil } - return fmt.Sprintf("%s = '%s'", s.fieldFormat(c.Field), c.Value), nil + return fmt.Sprintf("%s = '%s'", s.dimTransform(c.Field), c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll) @@ -144,7 +145,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = ">" } - start = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.Start) + start = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.Start) } if *c.End != "*" { @@ -154,7 +155,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = "<" } - end = fmt.Sprintf("%s %s %s", s.fieldFormat(c.Field), op, *c.End) + end = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.End) } return fmt.Sprintf("%s", strings.Join([]string{start, end}, " AND ")), nil From 760011af32bf2f8633c825586798963d13cd4b09 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 7 Mar 2025 15:29:16 +0800 Subject: [PATCH 024/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/influxdb/router_mock.go | 29 +++++--- .../internal/querystring/parser_test.go | 7 ++ pkg/unify-query/query/structured/query_ts.go | 2 +- .../query/structured/space_test.go | 6 +- pkg/unify-query/service/http/query_test.go | 5 +- pkg/unify-query/tsdb/bksql/format.go | 44 +++--------- pkg/unify-query/tsdb/bksql/instance.go | 10 +-- pkg/unify-query/tsdb/bksql/instance_test.go | 44 ++++++++---- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 67 ++++++++++++------- .../tsdb/bksql/sqlExpr/base_test.go | 2 +- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 43 ++++++------ .../tsdb/bksql/sqlExpr/doris_test.go | 11 ++- .../tsdb/elasticsearch/query_string.go | 32 ++++++--- 13 files changed, 178 insertions(+), 124 deletions(-) diff --git a/pkg/unify-query/influxdb/router_mock.go b/pkg/unify-query/influxdb/router_mock.go index ecec10e89..df883a7df 100644 --- a/pkg/unify-query/influxdb/router_mock.go +++ b/pkg/unify-query/influxdb/router_mock.go @@ -195,6 +195,7 @@ func MockSpaceRouter(ctx context.Context) { Measurement: "kubelet_info", BcsClusterID: "BCS-K8S-00000", MeasurementType: redis.BkSplitMeasurement, + StorageType: consul.VictoriaMetricsStorageType, DataLabel: "kubelet_info", }, "system.cpu_summary": &ir.ResultTableDetail{ @@ -206,6 +207,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "cpu_summary", }, "system.cpu_detail": &ir.ResultTableDetail{ @@ -214,6 +216,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "100147_ieod_system_cpu_detail_raw", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "cpu_detail", }, "system.disk": &ir.ResultTableDetail{ @@ -222,6 +225,7 @@ func MockSpaceRouter(ctx context.Context) { VmRt: "100147_ieod_system_disk_raw", Fields: []string{"usage", "free"}, MeasurementType: redis.BKTraditionalMeasurement, + StorageType: consul.InfluxDBStorageType, DataLabel: "disk", }, ResultTableVM: &ir.ResultTableDetail{ @@ -231,6 +235,7 @@ func MockSpaceRouter(ctx context.Context) { Fields: vmFiedls, BcsClusterID: "BCS-K8S-00000", MeasurementType: redis.BkSplitMeasurement, + StorageType: consul.VictoriaMetricsStorageType, DataLabel: "vm", }, ResultTableInfluxDB: &ir.ResultTableDetail{ @@ -243,12 +248,14 @@ func MockSpaceRouter(ctx context.Context) { MeasurementType: redis.BkSplitMeasurement, ClusterName: "default", DataLabel: "influxdb", + StorageType: consul.InfluxDBStorageType, }, ResultTableEs: &ir.ResultTableDetail{ - StorageId: 3, - TableId: ResultTableEs, - DB: "es_index", - SourceType: "bkdata", + StorageId: 3, + TableId: ResultTableEs, + DB: "es_index", + SourceType: "bkdata", + StorageType: consul.ElasticsearchStorageType, StorageClusterRecords: []ir.Record{ { StorageID: 3, @@ -264,14 +271,16 @@ func MockSpaceRouter(ctx context.Context) { DataLabel: "es", }, ResultTableBkSQL: &ir.ResultTableDetail{ - StorageId: 4, - TableId: ResultTableBkSQL, - DataLabel: "bksql", + StorageId: 4, + TableId: ResultTableBkSQL, + DataLabel: "bksql", + StorageType: consul.BkSqlStorageType, }, ResultTableBkBaseEs: &ir.ResultTableDetail{ - SourceType: "bkdata", - DB: "es_index", - DataLabel: "bkbase_es", + SourceType: "bkdata", + DB: "es_index", + DataLabel: "bkbase_es", + StorageType: consul.ElasticsearchStorageType, }, }, nil, ir.DataLabelToResultTable{ diff --git a/pkg/unify-query/internal/querystring/parser_test.go b/pkg/unify-query/internal/querystring/parser_test.go index fa7187264..045afe09b 100644 --- a/pkg/unify-query/internal/querystring/parser_test.go +++ b/pkg/unify-query/internal/querystring/parser_test.go @@ -267,6 +267,13 @@ func TestParser(t *testing.T) { }, }, }, + "start": { + q: "a: >100", + e: &NumberRangeExpr{ + Field: "a", + Start: pointer("100"), + }, + }, } metadata.InitMetadata() diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index b793f66ca..a6d9b47c9 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -860,7 +860,7 @@ func (q *Query) BuildMetadataQuery( } if query.StorageType == "" { - return nil, fmt.Errorf("storageType is empty with %d", query.StorageID) + return nil, fmt.Errorf("storageType is empty with %v", query.StorageID) } query.Measurement = measurement diff --git a/pkg/unify-query/query/structured/space_test.go b/pkg/unify-query/query/structured/space_test.go index a6cea12bf..830f83461 100644 --- a/pkg/unify-query/query/structured/space_test.go +++ b/pkg/unify-query/query/structured/space_test.go @@ -43,17 +43,17 @@ func TestSpaceFilter_NewTsDBs(t *testing.T) { "test_1": { fieldName: "kube_node_info", tableID: "", - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"influxdb"}]`, }, "test_2_regex": { fieldName: "kubelet_.+", isRegexp: true, - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"influxdb"},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, "test_3_regex": { fieldName: "container_.+", isRegexp: true, - expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false}]`, + expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, } diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 6acb5ceb2..def3ed702 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -2453,13 +2453,10 @@ func TestQueryTsClusterMetrics(t *testing.T) { assert.Nil(t, err) res, err := QueryTsClusterMetrics(ctx, query) - t.Logf("QueryTsClusterMetrics error: %+v", err) assert.Nil(t, err) out, err := json.Marshal(res) - actual := string(out) assert.Nil(t, err) - fmt.Printf("ActualResult: %v\n", actual) - assert.Equal(t, c.result, actual) + assert.JSONEq(t, c.result, string(out)) }) } } diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 2d0ed41ea..0dc0f24ba 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -33,24 +33,19 @@ const ( startTime = "_startTime_" endTime = "_endTime_" theDate = "thedate" - - timeStamp = "_timestamp_" - value = "_value_" - - FieldValue = "_value" - FieldTime = "_time" ) var ( internalDimension = map[string]struct{}{ - value: {}, - timeStamp: {}, dtEventTimeStamp: {}, dtEventTime: {}, localTime: {}, startTime: {}, endTime: {}, theDate: {}, + + sqlExpr.TimeStamp: {}, + sqlExpr.Value: {}, } ) @@ -165,7 +160,7 @@ func (f *QueryFactory) BuildWhere() (string, error) { } func (f *QueryFactory) SQL() (sql string, err error) { - selectString, groupString, err := f.expr.ParserAggregates(f.query.Aggregates) + selectFields, groupFields, orderFields, err := f.expr.ParserAggregatesAndOrders(f.query.Aggregates, f.orders) if err != nil { return } @@ -175,7 +170,7 @@ func (f *QueryFactory) SQL() (sql string, err error) { table += "." + f.query.Measurement } - sql += fmt.Sprintf("SELECT %s FROM %s", selectString, table) + sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), table) whereString, err := f.BuildWhere() if err != nil { return @@ -183,30 +178,13 @@ func (f *QueryFactory) SQL() (sql string, err error) { if whereString != "" { sql += " WHERE " + whereString } - if groupString != "" { - sql += " GROUP BY " + groupString + if len(groupFields) > 0 { + sql += " GROUP BY " + strings.Join(groupFields, ", ") } - orders := make([]string, 0) - for key, asc := range f.orders { - var orderField string - switch key { - case FieldValue: - orderField = f.query.Field - case FieldTime: - orderField = timeStamp - default: - orderField = key - } - ascName := "ASC" - if !asc { - ascName = "DESC" - } - orders = append(orders, fmt.Sprintf("`%s` %s", orderField, ascName)) - } - if len(orders) > 0 { - sort.Strings(orders) - sql += " ORDER BY " + strings.Join(orders, ", ") + if len(orderFields) > 0 { + sort.Strings(orderFields) + sql += " ORDER BY " + strings.Join(orderFields, ", ") } if f.query.From > 0 { sql += fmt.Sprintf(" OFFSET %d", f.query.From) @@ -268,7 +246,7 @@ func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) } // 获取时间戳，单位是毫秒 - if vtLong, ok = d[timeStamp]; !ok { + if vtLong, ok = d[sqlExpr.TimeStamp]; !ok { vtLong = 0 } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 59641356e..8fa91aa64 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -29,6 +29,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) const ( @@ -189,7 +190,7 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad } // 获取时间戳，单位是毫秒 - if vtLong, ok = d[timeStamp]; !ok { + if vtLong, ok = d[sqlExpr.TimeStamp]; !ok { vtLong = start.UnixMilli() } @@ -206,8 +207,8 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad } // 获取值 - if vvDouble, ok = d[value]; !ok { - return res, fmt.Errorf("dimension %s is emtpy", value) + if vvDouble, ok = d[sqlExpr.Value]; !ok { + return res, fmt.Errorf("dimension %s is emtpy", sqlExpr.Value) } if vvDouble == nil { @@ -219,7 +220,7 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad case float64: vv = vvDouble.(float64) default: - return res, fmt.Errorf("%s type is error %T, %v", value, vvDouble, vvDouble) + return res, fmt.Errorf("%s type is error %T, %v", sqlExpr.Value, vvDouble, vvDouble) } lbl := make([]prompb.Label, 0) @@ -333,7 +334,6 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star list[KeyIndex] = query.DB list[KeyTableID] = query.TableID list[KeyDataLabel] = query.DataLabel - list[FieldTime] = list[timeStamp] if query.HighLight.Enable { list[KeyHighLight] = "" diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 7a4d8d894..595bc478c 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -11,6 +11,7 @@ package bksql_test import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -44,15 +45,16 @@ func TestInstance_QueryRaw(t *testing.T) { field := "login_rate" tableID := db + ".default" - for idx, c := range []struct { + for name, c := range map[string]struct { query *metadata.Query expected string }{ - { + "query with in": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, DB: db, + DataLabel: db, MetricName: field, BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", OffsetInfo: metadata.OffSetInfo{Limit: 10}, @@ -66,14 +68,15 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":269,"timestamp":1730118660000},{"value":271,"timestamp":1730118480000},{"value":267,"timestamp":1730118540000},{"value":274,"timestamp":1730118600000},{"value":279,"timestamp":1730118420000}],"exemplars":null,"histograms":null}]`, + expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:32:00","_startTime_":"2024-10-28 20:31:00","_timestamp_":1730118660000,"_value_":269,"dtEventTime":"2024-10-28 20:31:00","dtEventTimeStamp":1730118660000,"localTime":"2024-10-28 20:32:03","login_rate":269,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:29:00","_startTime_":"2024-10-28 20:28:00","_timestamp_":1730118480000,"_value_":271,"dtEventTime":"2024-10-28 20:28:00","dtEventTimeStamp":1730118480000,"localTime":"2024-10-28 20:29:03","login_rate":271,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:30:00","_startTime_":"2024-10-28 20:29:00","_timestamp_":1730118540000,"_value_":267,"dtEventTime":"2024-10-28 20:29:00","dtEventTimeStamp":1730118540000,"localTime":"2024-10-28 20:30:02","login_rate":267,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:31:00","_startTime_":"2024-10-28 20:30:00","_timestamp_":1730118600000,"_value_":274,"dtEventTime":"2024-10-28 20:30:00","dtEventTimeStamp":1730118600000,"localTime":"2024-10-28 20:31:04","login_rate":274,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:28:00","_startTime_":"2024-10-28 20:27:00","_timestamp_":1730118420000,"_value_":279,"dtEventTime":"2024-10-28 20:27:00","dtEventTimeStamp":1730118420000,"localTime":"2024-10-28 20:28:03","login_rate":279,"namespace":"gz100","thedate":20241028}]`, }, - { + "count by namespace": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, DB: db, MetricName: field, + DataLabel: db, Aggregates: metadata.Aggregates{ { Name: "count", @@ -81,14 +84,15 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"bgp2"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"cq100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn0-new"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn1"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn10"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"nj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"njloadtest"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"pbe"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj101"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null}]`, + expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"bgp2"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"cq100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"gz100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn0-new"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn1"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn10"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"nj100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"njloadtest"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"pbe"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"tj100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"tj101"}]`, }, - { + "count with 1m": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, DB: db, MetricName: field, + DataLabel: db, Aggregates: metadata.Aggregates{ { Name: "count", @@ -96,10 +100,10 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"}],"samples":[{"value":11,"timestamp":1730118600000},{"value":11,"timestamp":1730118660000},{"value":11,"timestamp":1730118720000},{"value":11,"timestamp":1730118780000},{"value":11,"timestamp":1730118840000}],"exemplars":null,"histograms":null}]`, + expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118600000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118660000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118720000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118780000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118840000,"_value_":11}]`, }, } { - t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) if c.query.DB == "" { c.query.DB = db @@ -107,13 +111,27 @@ func TestInstance_QueryRaw(t *testing.T) { if c.query.Field == "" { c.query.Field = field } - ss := ins.QuerySeriesSet(ctx, c.query, start, end) - timeSeries, err := mock.SeriesSetToTimeSeries(ss) - if err != nil { - log.Fatalf(ctx, err.Error()) + dataCh := make(chan map[string]any) + + go func() { + defer func() { + close(dataCh) + }() + + _, err := ins.QueryRawData(ctx, c.query, start, end, dataCh) + assert.Nil(t, err) + }() + + list := make([]map[string]any, 0) + for d := range dataCh { + list = append(list, d) } - assert.Equal(t, c.expected, timeSeries.String()) + + actual, err := json.Marshal(list) + assert.Nil(t, err) + + assert.Equal(t, c.expected, string(actual)) }) } } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index adf06f874..086061fae 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -19,9 +19,12 @@ import ( ) const ( - selectAll = "*" - timeStamp = "_timestamp_" - value = "_value_" + SelectAll = "*" + TimeStamp = "_timestamp_" + Value = "_value_" + + FieldValue = "_value" + FieldTime = "_time" theDate = "thedate" ) @@ -44,8 +47,8 @@ type SQLExpr interface { ParserQueryString(qs string) (string, error) // ParserAllConditions 解析全量条件生成SQL条件表达式 ParserAllConditions(allConditions metadata.AllConditions) (string, error) - // ParserAggregates 解析聚合条件生成SQL条件表达式 - ParserAggregates(aggregates metadata.Aggregates) (string, string, error) + // ParserAggregatesAndOrders 解析聚合条件生成SQL条件表达式 + ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, error) } // SQL表达式注册管理相关变量 @@ -56,6 +59,13 @@ var ( ExprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 ) +func defaultTransformDimension(s string) string { + if s == "" { + return "" + } + return fmt.Sprintf("`%s`", s) +} + // GetSQLExpr 获取指定key的SQL表达式实现 // 参数： // @@ -70,13 +80,7 @@ func GetSQLExpr(key string) SQLExpr { if sqlExpr, ok := ExprMap[key]; ok { return sqlExpr } else { - return (&DefaultSQLExpr{}).WithTransformDimension(func(s string) string { - if s == "" { - return "" - } - - return fmt.Sprintf("`%s`", s) - }) + return (&DefaultSQLExpr{}).WithTransformDimension(defaultTransformDimension) } } @@ -138,11 +142,7 @@ func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { } // ParserAggregates 解析聚合函数，生成 select 和 group by 字段 -func (d *DefaultSQLExpr) ParserAggregates(aggregates metadata.Aggregates) (string, string, error) { - var ( - selectFields []string - groupByFields []string - ) +func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { valueField := d.dimTransform(d.valueField) for _, agg := range aggregates { @@ -153,28 +153,47 @@ func (d *DefaultSQLExpr) ParserAggregates(aggregates metadata.Aggregates) (strin } if valueField == "" { - valueField = selectAll + valueField = SelectAll } - selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, value)) + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", d.timeField, d.timeField, agg.Window.Milliseconds()) groupByFields = append(groupByFields, timeField) - selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, timeStamp)) + selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) + + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) } } if len(selectFields) == 0 { - selectFields = append(selectFields, selectAll) + selectFields = append(selectFields, SelectAll) if valueField != "" { - selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, value)) + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, Value)) } if d.timeField != "" { - selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, timeStamp)) + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, TimeStamp)) + } + } + + for key, asc := range orders { + var orderField string + switch key { + case FieldValue: + orderField = d.valueField + case FieldTime: + orderField = TimeStamp + default: + orderField = key + } + ascName := "ASC" + if !asc { + ascName = "DESC" } + orderByFields = append(orderByFields, fmt.Sprintf("`%s` %s", orderField, ascName)) } - return strings.Join(selectFields, ", "), strings.Join(groupByFields, ", "), nil + return } // ParserAllConditions 解析全量条件生成SQL条件表达式 diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go index 46f46b56b..7736190e5 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go @@ -20,7 +20,7 @@ import ( // TestParserAllConditions 测试全条件解析的主测试函数 func TestParserAllConditions(t *testing.T) { - d := &sqlExpr.DefaultSQLExpr{} + d := sqlExpr.GetSQLExpr("") t.Run("空条件测试", func(t *testing.T) { conditions := metadata.AllConditions{} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index b12702812..3fe87643d 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -37,6 +37,12 @@ func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { return s } +func (s *DorisSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr { + s.dimTransform = fn + s.DefaultSQLExpr.WithTransformDimension(fn) + return s +} + func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { expr, err := querystring.Parse(qs) if err != nil { @@ -50,7 +56,7 @@ func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { } func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { - return s.DefaultSQLExpr.WithTransformDimension(s.dimTransform).ParserAllConditions(allConditions) + return s.DefaultSQLExpr.ParserAllConditions(allConditions) } func (s *DorisSQLExpr) checkMatchALL(k string) bool { @@ -64,14 +70,6 @@ func (s *DorisSQLExpr) checkMatchALL(k string) bool { return false } -func (s *DorisSQLExpr) dimTransform(field string) string { - fs := strings.Split(field, ".") - if len(fs) > 1 { - return fmt.Sprintf("CAST(`%s`[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)) - } - return fmt.Sprintf("`%s`", field) -} - func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { var ( err error @@ -134,31 +132,28 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } - var ( - start string - end string - ) - if *c.Start != "*" { + var timeFilter []string + if c.Start != nil && *c.Start != "*" { var op string if c.IncludeStart { op = ">=" } else { op = ">" } - start = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.Start) + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.Start)) } - if *c.End != "*" { + if c.End != nil && *c.End != "*" { var op string if c.IncludeEnd { op = "<=" } else { op = "<" } - end = fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.End) + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.End)) } - return fmt.Sprintf("%s", strings.Join([]string{start, end}, " AND ")), nil + return fmt.Sprintf("%s", strings.Join(timeFilter, " AND ")), nil default: err = fmt.Errorf("expr type is not match %T", e) } @@ -167,5 +162,15 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } func init() { - Register(Doris, &DorisSQLExpr{}) + Register(Doris, (&DorisSQLExpr{}).WithTransformDimension(func(s string) string { + if s == "" { + return "" + } + + fs := strings.Split(s, ".") + if len(fs) > 1 { + return fmt.Sprintf("CAST(`%s`[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)) + } + return fmt.Sprintf("`%s`", s) + })) } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index a00939783..55428d493 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -89,7 +89,12 @@ func TestDorisSQLExpr_ParserQueryString(t *testing.T) { { name: "object field", input: "__ext.container_name: value", - want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", + want: "CAST(`__ext`[\"container_name\"] AS STRING) = 'value'", + }, + { + name: "start", + input: "a: >100", + want: "`a` > 100", }, } @@ -132,7 +137,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, }, }, - want: "CAST(object[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", + want: "CAST(`object`[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", }, { name: "test text field wildcard", @@ -146,7 +151,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, }, }, - want: `CAST(object["field"] AS STRING) LIKE '%partial%'`, + want: "CAST(`object`[\"field\"] AS STRING) LIKE '%partial%'", }, { name: "test OR condition", diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index e9ba5f390..7e815016c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -10,11 +10,13 @@ package elasticsearch import ( + "context" "fmt" elastic "github.com/olivere/elastic/v7" qs "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) type QueryString struct { @@ -45,6 +47,12 @@ func (s *QueryString) queryString(str string) elastic.Query { } func (s *QueryString) ToDSL() (elastic.Query, error) { + defer func() { + if r := recover(); r != nil { + log.Errorf(context.TODO(), "querystring(%s) todsl panic: %v", s.q, r) + } + }() + if s.q == "" || s.q == "*" { return nil, nil } @@ -123,15 +131,23 @@ func (s *QueryString) walk(expr qs.Expr) (elastic.Query, error) { } case *qs.NumberRangeExpr: q := elastic.NewRangeQuery(c.Field) - if c.IncludeStart { - q.Gte(*c.Start) - } else { - q.Gt(*c.Start) + if c.Start == nil && c.End == nil { + return nil, fmt.Errorf("start and end is nil") } - if c.IncludeEnd { - q.Lte(*c.End) - } else { - q.Lt(*c.End) + + if c.Start != nil { + if c.IncludeStart { + q.Gte(*c.Start) + } else { + q.Gt(*c.Start) + } + } + if c.End != nil { + if c.IncludeEnd { + q.Lte(*c.End) + } else { + q.Lt(*c.End) + } } s.check(c.Field) leftQ = q From 80fd646d1c50cd1bc4e02f50f85ee290013ad710 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 7 Mar 2025 17:56:27 +0800 Subject: [PATCH 025/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 5 +- pkg/unify-query/tsdb/bksql/instance_test.go | 83 ++++++++++++++++----- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 78 +++++++++++++++---- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 40 ++++++---- 4 files changed, 156 insertions(+), 50 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 0dc0f24ba..1ac88a9e4 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -85,7 +85,10 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { } fieldsMap := make(map[string]string) - f.expr = sqlExpr.GetSQLExpr(f.query.Measurement).WithFieldsMap(fieldsMap).WithInternalFields(f.timeField, query.Field) + f.expr = sqlExpr.GetSQLExpr(f.query.Measurement). + WithFieldsMap(fieldsMap). + WithInternalFields(f.timeField, query.Field). + WithEncode(metadata.GetPromDataFormat(ctx).EncodeFunc()) return f } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 595bc478c..dd1ed3434 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -300,6 +300,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { end time.Time query *metadata.Query expected string + err error }{ // 测试用例1: 无聚合函数的原始查询 { @@ -434,7 +435,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { "__ext.container_id": false, }, }, - expected: "SELECT `__ext.container_id`, `test`, COUNT(`__ext.container_id`) AS `_value_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `__ext.container_id` = '1234567890' GROUP BY `__ext.container_id`, `test` ORDER BY `__ext.container_id` DESC, `timestamp` ASC", + err: fmt.Errorf("query is not support object with __ext.container_id"), }, // 测试用例8: doris 处理 object 字段 @@ -448,9 +449,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { AllConditions: metadata.AllConditions{ { { - DimensionName: "__ext.container_id", + DimensionName: "__ext.io_kubernetes_workload_name", Operator: metadata.ConditionEqual, - Value: []string{"5782c24d05ae0eceb3074d6ae492e7b027b7310711a4b63392eacd9314fc9a83"}, + Value: []string{"bkm-daemonset-worker"}, }, { DimensionName: "bk_host_id", @@ -462,15 +463,53 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { Aggregates: metadata.Aggregates{ { Name: "count", - Dimensions: []string{"__ext.container_id", "bk_host_id"}, + Dimensions: []string{"__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type"}, }, }, Orders: map[string]bool{ - "timestamp": true, - "__ext.container_id": false, + "__ext.io_kubernetes_workload_name": false, }, }, - expected: "", + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", + }, + // 测试用例8: doris 处理 object 字段 + 时间聚合 + { + name: "default multiple order fields and time aggregate", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "__ext.io_kubernetes_workload_name", + Operator: metadata.ConditionEqual, + Value: []string{"bkm-daemonset-worker"}, + }, + { + DimensionName: "bk_host_id", + Operator: metadata.ConditionEqual, + Value: []string{"267730"}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type"}, + Window: time.Minute, + }, + }, + Orders: map[string]bool{ + "__ext.io_kubernetes_workload_name": false, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, } @@ -491,9 +530,13 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 条件解析验证 if len(tc.query.AllConditions) > 0 { condition, err := sqlExpr.GetSQLExpr(tc.query.Measurement).WithFieldsMap(nil).ParserAllConditions(tc.query.AllConditions) - assert.Nil(t, err) - if err == nil { - assert.NotEmpty(t, condition, "Parsed condition should not be empty") + if tc.err != nil { + assert.Equal(t, tc.err, err) + } else { + assert.Nil(t, err) + if err == nil { + assert.NotEmpty(t, condition, "Parsed condition should not be empty") + } } } @@ -501,15 +544,19 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { fact := bksql.NewQueryFactory(ctx, tc.query).WithRangeTime(start, end) generatedSQL, err := fact.SQL() - assert.Nil(t, err) - if err == nil { - assert.Equal(t, tc.expected, generatedSQL) - } + if tc.err != nil { + assert.Equal(t, tc.err, err) + } else { + assert.Nil(t, err) + if err == nil { + assert.Equal(t, tc.expected, generatedSQL) - // 验证时间条件 - if tc.start.IsZero() && tc.end.IsZero() { - assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) - assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` < %d", baseEnd.UnixMilli())) + // 验证时间条件 + if tc.start.IsZero() && tc.end.IsZero() { + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` < %d", baseEnd.UnixMilli())) + } + } } }) } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 086061fae..566eecfe6 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -39,8 +39,10 @@ var ( type SQLExpr interface { // WithFieldsMap 设置字段类型 WithFieldsMap(fieldsMap map[string]string) SQLExpr - // WithTransformDimension 设置字段转换方法 - WithTransformDimension(func(string) string) SQLExpr + // WithDimensionTransform 设置标签字段转换方法 + WithDimensionTransform(func(string) (string, error)) SQLExpr + // WithEncode 字段转换方法 + WithEncode(func(string) string) SQLExpr // WithInternalFields 设置内部字段 WithInternalFields(timeField, valueField string) SQLExpr // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 @@ -59,11 +61,16 @@ var ( ExprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 ) -func defaultTransformDimension(s string) string { +func defaultDimensionTransform(s string) (string, error) { if s == "" { - return "" + return "", nil + } + fs := strings.Split(s, ".") + if len(fs) > 1 { + return "", fmt.Errorf("query is not support object with %s", s) } - return fmt.Sprintf("`%s`", s) + + return fmt.Sprintf("`%s`", s), nil } // GetSQLExpr 获取指定key的SQL表达式实现 @@ -80,7 +87,8 @@ func GetSQLExpr(key string) SQLExpr { if sqlExpr, ok := ExprMap[key]; ok { return sqlExpr } else { - return (&DefaultSQLExpr{}).WithTransformDimension(defaultTransformDimension) + return (&DefaultSQLExpr{}). + WithDimensionTransform(defaultDimensionTransform) } } @@ -109,8 +117,12 @@ func UnRegister(key string) { // DefaultSQLExpr SQL表达式默认实现 type DefaultSQLExpr struct { - dimTransform func(string) string - fieldsMap map[string]string + timeFieldTransform func(string, time.Duration) string + dimTransform func(string) (string, error) + + encodeFunc func(string) string + + fieldsMap map[string]string timeField string valueField string @@ -125,30 +137,54 @@ func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExp return d } +func (d *DefaultSQLExpr) WithEncode(fn func(string) string) SQLExpr { + d.encodeFunc = fn + return d +} + func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { d.fieldsMap = fieldsMap return d } -// WithTransformDimension 实现设置字段转换方法 -func (d *DefaultSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr { +// WithDimensionTransform 实现设置字段转换方法 +func (d *DefaultSQLExpr) WithDimensionTransform(fn func(string) (string, error)) SQLExpr { d.dimTransform = fn return d } +func (d *DefaultSQLExpr) WithTimeFieldTransform(fn func(string, time.Duration) string) SQLExpr { + d.timeFieldTransform = fn + return d +} + // ParserQueryString 解析查询字符串（当前实现返回空） func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { return "", nil } -// ParserAggregates 解析聚合函数，生成 select 和 group by 字段 +// ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { - valueField := d.dimTransform(d.valueField) + valueField, err := d.dimTransform(d.valueField) + if err != nil { + return + } for _, agg := range aggregates { for _, dim := range agg.Dimensions { - dim = d.dimTransform(dim) - selectFields = append(selectFields, dim) + var alias string + if d.encodeFunc != nil { + alias = d.encodeFunc(dim) + } else { + alias = dim + } + + dim, err = d.dimTransform(dim) + if err != nil { + return + } + alias = fmt.Sprintf("%s AS `%s`", dim, alias) + selectFields = append(selectFields, alias) groupByFields = append(groupByFields, dim) } @@ -159,6 +195,7 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate if agg.Window > 0 { timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", d.timeField, d.timeField, agg.Window.Milliseconds()) + groupByFields = append(groupByFields, timeField) selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) @@ -177,6 +214,11 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate } for key, asc := range orders { + key, err = d.dimTransform(key) + if err != nil { + return + } + var orderField string switch key { case FieldValue: @@ -186,11 +228,12 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate default: orderField = key } + ascName := "ASC" if !asc { ascName = "DESC" } - orderByFields = append(orderByFields, fmt.Sprintf("`%s` %s", orderField, ascName)) + orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) } return @@ -254,7 +297,10 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro val string ) - key = d.dimTransform(c.DimensionName) + key, err := d.dimTransform(c.DimensionName) + if err != nil { + return "", err + } // 根据操作符类型生成不同的SQL表达式 switch c.Operator { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 3fe87643d..a8fce4601 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -20,6 +20,8 @@ import ( const ( Doris = "doris" DorisTypeText = "text" + + ShardKey = "__shard_key__" ) type DorisSQLExpr struct { @@ -37,12 +39,6 @@ func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { return s } -func (s *DorisSQLExpr) WithTransformDimension(fn func(string) string) SQLExpr { - s.dimTransform = fn - s.DefaultSQLExpr.WithTransformDimension(fn) - return s -} - func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { expr, err := querystring.Parse(qs) if err != nil { @@ -114,24 +110,38 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } - return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil + field, err := s.dimTransform(c.Field) + if err != nil { + return "", err + } + + return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil case *querystring.MatchExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) return "", err } + field, err := s.dimTransform(c.Field) + if err != nil { + return "", err + } if s.checkMatchALL(c.Field) { - return fmt.Sprintf("%s LIKE '%%%s%%'", s.dimTransform(c.Field), c.Value), nil + return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil } - return fmt.Sprintf("%s = '%s'", s.dimTransform(c.Field), c.Value), nil + return fmt.Sprintf("%s = '%s'", field, c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { err = fmt.Errorf(Doris + " " + ErrorMatchAll) return "", err } + field, err := s.dimTransform(c.Field) + if err != nil { + return "", err + } + var timeFilter []string if c.Start != nil && *c.Start != "*" { var op string @@ -140,7 +150,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = ">" } - timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.Start)) + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", field, op, *c.Start)) } if c.End != nil && *c.End != "*" { @@ -150,7 +160,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } else { op = "<" } - timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", s.dimTransform(c.Field), op, *c.End)) + timeFilter = append(timeFilter, fmt.Sprintf("%s %s %s", field, op, *c.End)) } return fmt.Sprintf("%s", strings.Join(timeFilter, " AND ")), nil @@ -162,15 +172,15 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { } func init() { - Register(Doris, (&DorisSQLExpr{}).WithTransformDimension(func(s string) string { + Register(Doris, (&DorisSQLExpr{}).WithDimensionTransform(func(s string) (string, error) { if s == "" { - return "" + return "", nil } fs := strings.Split(s, ".") if len(fs) > 1 { - return fmt.Sprintf("CAST(`%s`[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)) + return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], "][")), nil } - return fmt.Sprintf("`%s`", s) + return fmt.Sprintf("`%s`", s), nil })) } From ffd4d87c6301b4c72e2ada9e88f076c5e0d1a189 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 7 Mar 2025 20:22:28 +0800 Subject: [PATCH 026/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format_test.go | 6 +- pkg/unify-query/tsdb/bksql/instance_test.go | 63 +++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 71 ++--- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 269 +++++++++++++++--- .../tsdb/bksql/sqlExpr/doris_test.go | 2 +- 5 files changed, 299 insertions(+), 112 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 7f0a98383..e338e0771 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -59,7 +59,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 0, Orders: metadata.Orders{"ip": true}, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", }, "sum-with-promql-1": { query: &metadata.Query{ @@ -104,7 +104,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, "count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -136,7 +136,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index dd1ed3434..38dcef853 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -509,7 +509,55 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), __shard_key__ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + }, + // 测试用例8: doris 处理 object 字段 + 时间聚合 5m + { + name: "default multiple order fields and time aggregate 5m", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{}, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name"}, + Window: time.Minute * 5, + }, + }, + Orders: map[string]bool{ + "__ext.io_kubernetes_workload_name": false, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "", + }, + // 测试用例8: doris 处理 object 字段 + 时间聚合 15s + { + name: "default multiple order fields and time aggregate 15s", + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: sqlExpr.Doris, + Field: "__ext.container_id", + Size: 3, + AllConditions: metadata.AllConditions{}, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"__ext.io_kubernetes_workload_name"}, + Window: time.Second * 15, + }, + }, + Orders: map[string]bool{ + "__ext.io_kubernetes_workload_name": false, + }, + }, + start: time.Unix(1741334700, 0), + end: time.Unix(1741335000, 0), + expected: "", }, } @@ -527,19 +575,6 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { end = baseEnd } - // 条件解析验证 - if len(tc.query.AllConditions) > 0 { - condition, err := sqlExpr.GetSQLExpr(tc.query.Measurement).WithFieldsMap(nil).ParserAllConditions(tc.query.AllConditions) - if tc.err != nil { - assert.Equal(t, tc.err, err) - } else { - assert.Nil(t, err) - if err == nil { - assert.NotEmpty(t, condition, "Parsed condition should not be empty") - } - } - } - // SQL生成验证 fact := bksql.NewQueryFactory(ctx, tc.query).WithRangeTime(start, end) generatedSQL, err := fact.SQL() diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 566eecfe6..674af9c02 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -13,7 +13,6 @@ import ( "fmt" "strings" "sync" - "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" ) @@ -39,8 +38,6 @@ var ( type SQLExpr interface { // WithFieldsMap 设置字段类型 WithFieldsMap(fieldsMap map[string]string) SQLExpr - // WithDimensionTransform 设置标签字段转换方法 - WithDimensionTransform(func(string) (string, error)) SQLExpr // WithEncode 字段转换方法 WithEncode(func(string) string) SQLExpr // WithInternalFields 设置内部字段 @@ -58,21 +55,9 @@ var ( _ SQLExpr = (*DefaultSQLExpr)(nil) // 接口实现检查 lock sync.RWMutex // 读写锁用于并发安全 - ExprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 + exprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 ) -func defaultDimensionTransform(s string) (string, error) { - if s == "" { - return "", nil - } - fs := strings.Split(s, ".") - if len(fs) > 1 { - return "", fmt.Errorf("query is not support object with %s", s) - } - - return fmt.Sprintf("`%s`", s), nil -} - // GetSQLExpr 获取指定key的SQL表达式实现 // 参数： // @@ -84,11 +69,10 @@ func defaultDimensionTransform(s string) (string, error) { func GetSQLExpr(key string) SQLExpr { lock.RLock() defer lock.RUnlock() - if sqlExpr, ok := ExprMap[key]; ok { + if sqlExpr, ok := exprMap[key]; ok { return sqlExpr } else { - return (&DefaultSQLExpr{}). - WithDimensionTransform(defaultDimensionTransform) + return &DefaultSQLExpr{} } } @@ -100,7 +84,7 @@ func GetSQLExpr(key string) SQLExpr { func Register(key string, sqlExpr SQLExpr) { lock.Lock() defer lock.Unlock() - ExprMap[key] = sqlExpr + exprMap[key] = sqlExpr } // UnRegister 注销指定key的SQL表达式实现 @@ -110,25 +94,19 @@ func Register(key string, sqlExpr SQLExpr) { func UnRegister(key string) { lock.Lock() defer lock.Unlock() - if _, ok := ExprMap[key]; ok { - delete(ExprMap, key) + if _, ok := exprMap[key]; ok { + delete(exprMap, key) } } // DefaultSQLExpr SQL表达式默认实现 type DefaultSQLExpr struct { - timeFieldTransform func(string, time.Duration) string - dimTransform func(string) (string, error) - encodeFunc func(string) string fieldsMap map[string]string timeField string valueField string - - start time.Time - end time.Time } func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { @@ -147,15 +125,16 @@ func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { return d } -// WithDimensionTransform 实现设置字段转换方法 -func (d *DefaultSQLExpr) WithDimensionTransform(fn func(string) (string, error)) SQLExpr { - d.dimTransform = fn - return d -} +func (d *DefaultSQLExpr) dimTransform(s string) (string, error) { + if s == "" { + return "", nil + } + fs := strings.Split(s, ".") + if len(fs) > 1 { + return "", fmt.Errorf("query is not support object with %s", s) + } -func (d *DefaultSQLExpr) WithTimeFieldTransform(fn func(string, time.Duration) string) SQLExpr { - d.timeFieldTransform = fn - return d + return fmt.Sprintf("`%s`", s), nil } // ParserQueryString 解析查询字符串（当前实现返回空） @@ -172,19 +151,11 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate for _, agg := range aggregates { for _, dim := range agg.Dimensions { - var alias string - if d.encodeFunc != nil { - alias = d.encodeFunc(dim) - } else { - alias = dim - } - dim, err = d.dimTransform(dim) if err != nil { return } - alias = fmt.Sprintf("%s AS `%s`", dim, alias) - selectFields = append(selectFields, alias) + selectFields = append(selectFields, dim) groupByFields = append(groupByFields, dim) } @@ -214,11 +185,6 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate } for key, asc := range orders { - key, err = d.dimTransform(key) - if err != nil { - return - } - var orderField string switch key { case FieldValue: @@ -229,6 +195,11 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate orderField = key } + orderField, err = d.dimTransform(orderField) + if err != nil { + return + } + ascName := "ASC" if !asc { ascName = "DESC" diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index a8fce4601..9d058889b 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -25,21 +25,33 @@ const ( ) type DorisSQLExpr struct { - DefaultSQLExpr + encodeFunc func(string) string + + timeField string + valueField string fieldsMap map[string]string } var _ SQLExpr = (*DorisSQLExpr)(nil) -func (s *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { - s.fieldsMap = fieldsMap +func (d *DorisSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { + d.timeField = timeField + d.valueField = valueField + return d +} - s.DefaultSQLExpr.WithFieldsMap(fieldsMap) - return s +func (d *DorisSQLExpr) WithEncode(fn func(string) string) SQLExpr { + d.encodeFunc = fn + return d } -func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { +func (d *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { + d.fieldsMap = fieldsMap + return d +} + +func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { expr, err := querystring.Parse(qs) if err != nil { return "", err @@ -48,16 +60,193 @@ func (s *DorisSQLExpr) ParserQueryString(qs string) (string, error) { return "", nil } - return s.walk(expr) + return d.walk(expr) } -func (s *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { - return s.DefaultSQLExpr.ParserAllConditions(allConditions) +// ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 +func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { + valueField, _ := d.dimTransform(d.valueField) + + for _, agg := range aggregates { + for _, dim := range agg.Dimensions { + var ( + isObject = false + + newDim string + selectAlias string + ) + newDim, isObject = d.dimTransform(dim) + if isObject && d.encodeFunc != nil { + selectAlias = fmt.Sprintf("%s AS `%s`", newDim, d.encodeFunc(dim)) + } else { + selectAlias = newDim + } + + selectFields = append(selectFields, selectAlias) + groupByFields = append(groupByFields, newDim) + } + + if valueField == "" { + valueField = SelectAll + } + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + + if agg.Window > 0 { + groupByFields = append(groupByFields, ShardKey) + selectFields = append(selectFields, fmt.Sprintf("%s * 60 AS `%s`", ShardKey, TimeStamp)) + + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) + } + } + + if len(selectFields) == 0 { + selectFields = append(selectFields, SelectAll) + if valueField != "" { + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, Value)) + } + if d.timeField != "" { + selectFields = append(selectFields, fmt.Sprintf("`%s` AS `%s`", d.timeField, TimeStamp)) + } + } + + for key, asc := range orders { + + var orderField string + switch key { + case FieldValue: + orderField = d.valueField + case FieldTime: + orderField = TimeStamp + default: + orderField = key + } + + orderField, _ = d.dimTransform(orderField) + ascName := "ASC" + if !asc { + ascName = "DESC" + } + orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) + } + + return } -func (s *DorisSQLExpr) checkMatchALL(k string) bool { - if s.fieldsMap != nil { - if t, ok := s.fieldsMap[k]; ok { +func (d *DorisSQLExpr) ParserAllConditions(allConditions metadata.AllConditions) (string, error) { + var ( + orConditions []string + ) + + // 遍历所有OR条件组 + for _, conditions := range allConditions { + var andConditions []string + // 处理每个AND条件组 + for _, cond := range conditions { + buildCondition, err := d.buildCondition(cond) + if err != nil { + return "", err + } + if buildCondition != "" { + andConditions = append(andConditions, buildCondition) + } + } + // 合并AND条件 + if len(andConditions) > 0 { + orConditions = append(orConditions, strings.Join(andConditions, " AND ")) + } + } + + // 处理最终OR条件组合 + if len(orConditions) > 0 { + if len(orConditions) == 1 { + return orConditions[0], nil + } + return fmt.Sprintf("(%s)", strings.Join(orConditions, " OR ")), nil + } + + return "", nil +} + +func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { + if len(c.Value) == 0 { + return "", nil + } + + var ( + key string + op string + val string + ) + + key, _ = d.dimTransform(c.DimensionName) + + // 根据操作符类型生成不同的SQL表达式 + switch c.Operator { + // 处理等于类操作符（=, IN） + case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) == 1 { + if c.IsWildcard { + op = "LIKE" + val = fmt.Sprintf("'%%%s%%'", c.Value[0]) + } else { + op = "=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } + } else { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理不等于类操作符（!=, NOT IN） + case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) == 1 { + op = "!=" + val = fmt.Sprintf("'%s'", c.Value[0]) + } else { + op = "NOT IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } + // 处理正则表达式匹配 + case metadata.ConditionRegEqual: + op = "REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) // 多个值用|连接 + case metadata.ConditionNotRegEqual: + op = "NOT REGEXP" + val = fmt.Sprintf("'%s'", strings.Join(c.Value, "|")) + // 处理数值比较操作符（>, >=, <, <=） + case metadata.ConditionGt: + op = ">" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionGte: + op = ">=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLt: + op = "<" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + case metadata.ConditionLte: + op = "<=" + if len(c.Value) != 1 { + return "", fmt.Errorf("operator %s only support 1 value", op) + } + val = c.Value[0] + default: + return "", fmt.Errorf("unknown operator %s", c.Operator) + } + + return fmt.Sprintf("%s %s %s", key, op, val), nil +} + +func (d *DorisSQLExpr) checkMatchALL(k string) bool { + if d.fieldsMap != nil { + if t, ok := d.fieldsMap[k]; ok { if t == DorisTypeText { return true } @@ -66,40 +255,40 @@ func (s *DorisSQLExpr) checkMatchALL(k string) bool { return false } -func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { +func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { var ( err error left string right string ) - if s == nil { + if d == nil { return "", nil } switch c := e.(type) { case *querystring.NotExpr: - left, err = s.walk(c.Expr) + left, err = d.walk(c.Expr) if err != nil { return "", err } return fmt.Sprintf("NOT (%s)", left), nil case *querystring.OrExpr: - left, err = s.walk(c.Left) + left, err = d.walk(c.Left) if err != nil { return "", err } - right, err = s.walk(c.Right) + right, err = d.walk(c.Right) if err != nil { return "", err } return fmt.Sprintf("(%s OR %s)", left, right), nil case *querystring.AndExpr: - left, err = s.walk(c.Left) + left, err = d.walk(c.Left) if err != nil { return "", err } - right, err = s.walk(c.Right) + right, err = d.walk(c.Right) if err != nil { return "", err } @@ -110,10 +299,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } - field, err := s.dimTransform(c.Field) - if err != nil { - return "", err - } + field, _ := d.dimTransform(c.Field) return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil case *querystring.MatchExpr: @@ -121,12 +307,9 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) return "", err } - field, err := s.dimTransform(c.Field) - if err != nil { - return "", err - } + field, _ := d.dimTransform(c.Field) - if s.checkMatchALL(c.Field) { + if d.checkMatchALL(c.Field) { return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil } @@ -136,11 +319,7 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { err = fmt.Errorf(Doris + " " + ErrorMatchAll) return "", err } - - field, err := s.dimTransform(c.Field) - if err != nil { - return "", err - } + field, _ := d.dimTransform(c.Field) var timeFilter []string if c.Start != nil && *c.Start != "*" { @@ -171,16 +350,18 @@ func (s *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return "", err } -func init() { - Register(Doris, (&DorisSQLExpr{}).WithDimensionTransform(func(s string) (string, error) { - if s == "" { - return "", nil - } +func (d *DorisSQLExpr) dimTransform(s string) (string, bool) { + if s == "" { + return "", false + } - fs := strings.Split(s, ".") - if len(fs) > 1 { - return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], "][")), nil - } - return fmt.Sprintf("`%s`", s), nil - })) + fs := strings.Split(s, ".") + if len(fs) > 1 { + return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], "][")), true + } + return fmt.Sprintf("`%s`", s), false +} + +func init() { + Register(Doris, &DorisSQLExpr{}) } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 55428d493..9fbb1b385 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -89,7 +89,7 @@ func TestDorisSQLExpr_ParserQueryString(t *testing.T) { { name: "object field", input: "__ext.container_name: value", - want: "CAST(`__ext`[\"container_name\"] AS STRING) = 'value'", + want: "CAST(__ext[\"container_name\"] AS STRING) = 'value'", }, { name: "start", From 8734f85b0aa4dd6129ff16bbf8b8827ce492303c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 12 Mar 2025 14:47:13 +0800 Subject: [PATCH 027/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format_test.go | 8 ++--- pkg/unify-query/tsdb/bksql/instance_test.go | 32 +++++++++---------- .../tsdb/bksql/sqlExpr/doris_test.go | 12 +++---- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index e338e0771..f3544a3d3 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -32,7 +32,7 @@ func TestNewSqlFactory(t *testing.T) { start time.Time end time.Time }{ - "sum-count_over_time-with-promql-1": { + "doris sum-count_over_time-with-promql-1": { query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -61,7 +61,7 @@ func TestNewSqlFactory(t *testing.T) { }, expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", }, - "sum-with-promql-1": { + "doris sum-with-promql-1": { query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -89,7 +89,7 @@ func TestNewSqlFactory(t *testing.T) { }, expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, - "count-with-count-promql-1": { + "doris count-with-count-promql-1": { query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -106,7 +106,7 @@ func TestNewSqlFactory(t *testing.T) { }, expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", }, - "count-with-count-promql-2": { + "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC // 2024-12-08 05:36:40 Asia/ShangHai start: time.Unix(1733607400, 0), diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 38dcef853..dae0390ef 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -304,7 +304,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }{ // 测试用例1: 无聚合函数的原始查询 { - name: "raw query without aggregation", + name: "mysql raw query without aggregation", query: &metadata.Query{ DB: "test_db", Field: "value", @@ -314,7 +314,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例2: 多聚合函数组合 { - name: "multiple aggregates", + name: "mysql multiple aggregates", query: &metadata.Query{ DB: "metrics_db", Field: "temperature", @@ -328,7 +328,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例3: 复杂条件组合 { - name: "complex conditions", + name: "mysql complex conditions", query: &metadata.Query{ DB: "security_logs", Field: "duration", @@ -352,7 +352,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例4: 多字段排序 { - name: "multiple order fields", + name: "mysql multiple order fields", query: &metadata.Query{ DB: "transaction_logs", Field: "amount", @@ -366,7 +366,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例5: 特殊字符转义 { - name: "special characters in fields", + name: "mysql special characters in fields", query: &metadata.Query{ DB: "special_metrics", Measurement: "select", // 保留字作为measurement @@ -380,7 +380,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例6: 零窗口时间 { - name: "zero window size", + name: "mysql zero window size", query: &metadata.Query{ DB: "time_series_data", Field: "value", @@ -396,7 +396,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例7: 跨多天的时间范围 { - name: "multi-day time range", + name: "mysql multi-day time range", start: crossDayStart, end: crossDayEnd, query: &metadata.Query{ @@ -411,7 +411,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 测试用例8: 默认处理 object 字段 { - name: "default multiple order fields", + name: "mysql default multiple order fields", query: &metadata.Query{ DB: "transaction_logs", Field: "__ext.container_id", @@ -438,9 +438,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { err: fmt.Errorf("query is not support object with __ext.container_id"), }, - // 测试用例8: doris 处理 object 字段 + // 测试用例9: doris 处理 object 字段 { - name: "default multiple order fields", + name: "doris default multiple order fields", query: &metadata.Query{ DB: "5000140_bklog_container_log_demo_analysis", Measurement: sqlExpr.Doris, @@ -474,9 +474,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { end: time.Unix(1741335000, 0), expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", }, - // 测试用例8: doris 处理 object 字段 + 时间聚合 + // 测试用例10: doris 处理 object 字段 + 时间聚合 { - name: "default multiple order fields and time aggregate", + name: "doris default multiple order fields and time aggregate", query: &metadata.Query{ DB: "5000140_bklog_container_log_demo_analysis", Measurement: sqlExpr.Doris, @@ -511,9 +511,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { end: time.Unix(1741335000, 0), expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), __shard_key__ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, - // 测试用例8: doris 处理 object 字段 + 时间聚合 5m + // 测试用例11: doris 处理 object 字段 + 时间聚合 5m { - name: "default multiple order fields and time aggregate 5m", + name: "doris default multiple order fields and time aggregate 5m", query: &metadata.Query{ DB: "5000140_bklog_container_log_demo_analysis", Measurement: sqlExpr.Doris, @@ -535,9 +535,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { end: time.Unix(1741335000, 0), expected: "", }, - // 测试用例8: doris 处理 object 字段 + 时间聚合 15s + // 测试用例12: doris 处理 object 字段 + 时间聚合 15s { - name: "default multiple order fields and time aggregate 15s", + name: "doris default multiple order fields and time aggregate 15s", query: &metadata.Query{ DB: "5000140_bklog_container_log_demo_analysis", Measurement: sqlExpr.Doris, diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 9fbb1b385..3e52ab0f3 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -122,7 +122,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { wantErr error }{ { - name: "test object field condition", + name: "doris test object field condition", condition: metadata.AllConditions{ { { @@ -140,7 +140,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { want: "CAST(`object`[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", }, { - name: "test text field wildcard", + name: "doris t8est text field wildcard", condition: metadata.AllConditions{ { { @@ -154,7 +154,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { want: "CAST(`object`[\"field\"] AS STRING) LIKE '%partial%'", }, { - name: "test OR condition", + name: "doris test OR condition", condition: metadata.AllConditions{ { { @@ -174,7 +174,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { want: "(`status` = 'running' OR `code` = '500')", }, { - name: "test numeric field without cast", + name: "doris test numeric field without cast", condition: metadata.AllConditions{ { { @@ -200,7 +200,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { want: "`env` IN ('prod', 'test')", }, { - name: "test empty value", + name: "doris test empty value", condition: metadata.AllConditions{ { { @@ -213,7 +213,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { want: "", }, { - name: "test invalid operator", + name: "doris test invalid operator", condition: metadata.AllConditions{ { { From 4a5d575e715e131c2854a28e9e926c8ce05eb24e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Mar 2025 09:35:34 +0800 Subject: [PATCH 028/104] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=20json=20?= =?UTF-8?q?=E5=87=BD=E6=95=B0=20#1010158081122355416?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/bkapi/bkapi.go | 3 +- pkg/unify-query/bkapi/bkapi_test.go | 2 +- pkg/unify-query/bkapi/bkdata.go | 3 +- pkg/unify-query/bkapi/bkdata_test.go | 8 ++-- pkg/unify-query/consul/bcs_info.go | 2 +- pkg/unify-query/consul/downsampled.go | 2 +- pkg/unify-query/consul/es.go | 3 +- pkg/unify-query/consul/influxdb.go | 3 +- pkg/unify-query/consul/router.go | 2 +- pkg/unify-query/consul/storage.go | 3 +- pkg/unify-query/curl/curl.go | 2 +- pkg/unify-query/es/alias.go | 2 +- pkg/unify-query/featureFlag/featureFlag.go | 2 +- pkg/unify-query/go.mod | 6 ++- pkg/unify-query/go.sum | 18 +++++--- .../influxdb/client/client_test.go | 2 +- pkg/unify-query/influxdb/decoder/json.go | 3 +- pkg/unify-query/influxdb/influxdb_router.go | 2 +- pkg/unify-query/influxdb/influxdb_test.go | 4 +- pkg/unify-query/influxdb/router_mock.go | 1 - .../influxdb/spacetsdb_router_test.go | 2 + pkg/unify-query/internal/json/json.go | 43 +++++++++++++++++++ pkg/unify-query/metadata/queries.go | 3 +- pkg/unify-query/metadata/struct.go | 2 +- pkg/unify-query/mock/function.go | 3 +- pkg/unify-query/mock/handler.go | 3 +- pkg/unify-query/query/es/es_test.go | 2 +- pkg/unify-query/query/infos/infos.go | 2 +- pkg/unify-query/query/promql/series.go | 2 +- pkg/unify-query/query/structured/query_ts.go | 2 +- .../query/structured/space_test.go | 2 +- pkg/unify-query/query/structured/utils.go | 2 +- pkg/unify-query/redis/router.go | 3 +- pkg/unify-query/service/http/api.go | 2 +- pkg/unify-query/service/http/api/api.go | 2 +- pkg/unify-query/service/http/check_handler.go | 2 +- pkg/unify-query/service/http/es.go | 2 +- pkg/unify-query/service/http/es_test.go | 2 +- pkg/unify-query/service/http/handler.go | 2 +- pkg/unify-query/service/http/handler_test.go | 2 +- pkg/unify-query/service/http/query.go | 2 +- pkg/unify-query/service/http/query_test.go | 25 +++++++++-- pkg/unify-query/service/redis/redis.go | 2 +- pkg/unify-query/tsdb/bksql/client.go | 2 +- .../tsdb/elasticsearch/agg_format.go | 3 +- pkg/unify-query/tsdb/elasticsearch/format.go | 2 +- .../tsdb/elasticsearch/format_test.go | 8 ++-- .../tsdb/elasticsearch/instance.go | 2 +- .../tsdb/elasticsearch/instance_test.go | 6 +-- .../tsdb/elasticsearch/query_string_test.go | 4 +- pkg/unify-query/tsdb/influxdb/instance.go | 2 +- .../tsdb/offlineDataArchive/instance.go | 2 +- pkg/unify-query/tsdb/redis/instance.go | 2 +- .../tsdb/victoriaMetrics/instance.go | 2 +- .../tsdb/victoriaMetrics/instance_test.go | 2 +- 55 files changed, 151 insertions(+), 73 deletions(-) create mode 100644 pkg/unify-query/internal/json/json.go diff --git a/pkg/unify-query/bkapi/bkapi.go b/pkg/unify-query/bkapi/bkapi.go index 32220b99f..e48786899 100644 --- a/pkg/unify-query/bkapi/bkapi.go +++ b/pkg/unify-query/bkapi/bkapi.go @@ -10,11 +10,12 @@ package bkapi import ( - "encoding/json" "fmt" "sync" "github.com/spf13/viper" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) const ( diff --git a/pkg/unify-query/bkapi/bkapi_test.go b/pkg/unify-query/bkapi/bkapi_test.go index 1d8175613..09dccceff 100644 --- a/pkg/unify-query/bkapi/bkapi_test.go +++ b/pkg/unify-query/bkapi/bkapi_test.go @@ -32,6 +32,6 @@ func TestGetBkAPI(t *testing.T) { assert.Equal(t, map[string]string{ "Content-Type": "application/json", - "X-Bkapi-Authorization": `{"bk_app_code":"bk_code","bk_app_secret":"bk_secret","bk_username":"admin"}`, + "X-Bkapi-Authorization": `{"bk_username":"admin","bk_app_code":"bk_code","bk_app_secret":"bk_secret"}`, }, headers) } diff --git a/pkg/unify-query/bkapi/bkdata.go b/pkg/unify-query/bkapi/bkdata.go index d3be406db..a80dd3529 100644 --- a/pkg/unify-query/bkapi/bkdata.go +++ b/pkg/unify-query/bkapi/bkdata.go @@ -10,12 +10,13 @@ package bkapi import ( - "encoding/json" "fmt" "strings" "sync" "github.com/spf13/viper" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) const ( diff --git a/pkg/unify-query/bkapi/bkdata_test.go b/pkg/unify-query/bkapi/bkdata_test.go index 456b6133e..0ad76ab39 100644 --- a/pkg/unify-query/bkapi/bkdata_test.go +++ b/pkg/unify-query/bkapi/bkdata_test.go @@ -25,11 +25,9 @@ func TestGetDataAuth(t *testing.T) { "Content-Type": "application/json", }) - assert.Equal(t, map[string]string{ - "Content-Type": "application/json", - "X-Bkapi-Authorization": `{"bk_app_code":"bk_code","bk_app_secret":"bk_secret","bk_username":"admin"}`, - "X-Bkbase-Authorization": `{"bk_app_code":"bk_code","bk_username":"admin","bkdata_authentication_method":"token","bkdata_data_token":"123456"}`, - }, headers) + assert.Equal(t, "application/json", headers["Content-Type"]) + assert.JSONEq(t, `{"bk_app_code":"bk_code","bk_app_secret":"bk_secret","bk_username":"admin"}`, headers["X-Bkapi-Authorization"]) + assert.JSONEq(t, `{"bk_username":"admin","bkdata_data_token":"123456","bkdata_authentication_method":"token","bk_app_code":"bk_code"}`, headers["X-Bkbase-Authorization"]) } func TestGetDataUrl(t *testing.T) { diff --git a/pkg/unify-query/consul/bcs_info.go b/pkg/unify-query/consul/bcs_info.go index 539b8c8fa..446bc4239 100644 --- a/pkg/unify-query/consul/bcs_info.go +++ b/pkg/unify-query/consul/bcs_info.go @@ -11,12 +11,12 @@ package consul import ( "context" - "encoding/json" "regexp" "sync" "github.com/hashicorp/consul/api" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/consul/downsampled.go b/pkg/unify-query/consul/downsampled.go index a644b67f8..433ac0ff0 100644 --- a/pkg/unify-query/consul/downsampled.go +++ b/pkg/unify-query/consul/downsampled.go @@ -11,13 +11,13 @@ package consul import ( "context" - "encoding/json" "fmt" "strings" "sync" "github.com/hashicorp/consul/api" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/consul/es.go b/pkg/unify-query/consul/es.go index c9d8899f7..9d6aa08ba 100644 --- a/pkg/unify-query/consul/es.go +++ b/pkg/unify-query/consul/es.go @@ -11,11 +11,12 @@ package consul import ( "context" - "encoding/json" "fmt" "strings" "github.com/hashicorp/consul/api" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) const ( diff --git a/pkg/unify-query/consul/influxdb.go b/pkg/unify-query/consul/influxdb.go index 3aa88c8ab..73c86c087 100644 --- a/pkg/unify-query/consul/influxdb.go +++ b/pkg/unify-query/consul/influxdb.go @@ -11,11 +11,12 @@ package consul import ( "context" - "encoding/json" "fmt" "strings" "github.com/hashicorp/consul/api" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) const ( diff --git a/pkg/unify-query/consul/router.go b/pkg/unify-query/consul/router.go index af45dc8aa..63d47d0ce 100644 --- a/pkg/unify-query/consul/router.go +++ b/pkg/unify-query/consul/router.go @@ -11,7 +11,6 @@ package consul import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -19,6 +18,7 @@ import ( "github.com/hashicorp/consul/api" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/consul/storage.go b/pkg/unify-query/consul/storage.go index 2fb19f47d..31d7a0ec1 100644 --- a/pkg/unify-query/consul/storage.go +++ b/pkg/unify-query/consul/storage.go @@ -11,11 +11,12 @@ package consul import ( "context" - "encoding/json" "fmt" "strings" "github.com/hashicorp/consul/api" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) var ( diff --git a/pkg/unify-query/curl/curl.go b/pkg/unify-query/curl/curl.go index 7c0e11d5f..74428bef0 100644 --- a/pkg/unify-query/curl/curl.go +++ b/pkg/unify-query/curl/curl.go @@ -12,7 +12,6 @@ package curl import ( "bytes" "context" - "encoding/json" "fmt" "io" "net/http" @@ -21,6 +20,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" ) diff --git a/pkg/unify-query/es/alias.go b/pkg/unify-query/es/alias.go index 087553b62..f6ca10dec 100644 --- a/pkg/unify-query/es/alias.go +++ b/pkg/unify-query/es/alias.go @@ -11,10 +11,10 @@ package es import ( "context" - "encoding/json" "strconv" "sync" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/featureFlag/featureFlag.go b/pkg/unify-query/featureFlag/featureFlag.go index 51e1a671b..3bd10a9e6 100644 --- a/pkg/unify-query/featureFlag/featureFlag.go +++ b/pkg/unify-query/featureFlag/featureFlag.go @@ -11,13 +11,13 @@ package featureFlag import ( "context" - "encoding/json" "sync" ffclient "github.com/thomaspoignant/go-feature-flag" "github.com/thomaspoignant/go-feature-flag/exporter" "github.com/thomaspoignant/go-feature-flag/ffuser" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/go.mod b/pkg/unify-query/go.mod index 2529a42f2..6ba8183c8 100755 --- a/pkg/unify-query/go.mod +++ b/pkg/unify-query/go.mod @@ -9,6 +9,7 @@ require ( github.com/VictoriaMetrics/metricsql v0.69.0 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef github.com/bytedance/go-querystring-parser v0.0.0-20230310053818-dcfffcaee797 + github.com/bytedance/sonic v1.12.3 github.com/dgraph-io/ristretto v0.1.1 github.com/dominikbraun/graph v0.23.0 github.com/elastic/go-elasticsearch/v8 v8.0.0-20210317102009-a9d74cec0186 @@ -68,11 +69,12 @@ require ( github.com/aws/aws-sdk-go v1.44.280 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8 // indirect diff --git a/pkg/unify-query/go.sum b/pkg/unify-query/go.sum index 0b168a2ac..0dc80e0d6 100644 --- a/pkg/unify-query/go.sum +++ b/pkg/unify-query/go.sum @@ -136,9 +136,11 @@ github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+Wji github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/go-querystring-parser v0.0.0-20230310053818-dcfffcaee797 h1:web4OhRr3gxnebV/efpzprqZAFifhdNAd+T6UFpN2uk= github.com/bytedance/go-querystring-parser v0.0.0-20230310053818-dcfffcaee797/go.mod h1:ql3k2VAOCO+gkqC8GwuSGPFfVSumPGSqz3urT/twpaQ= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= +github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/c-bata/go-prompt v0.2.2 h1:uyKRz6Z6DUyj49QVijyM339UJV9yhbr70gESwbNU3e0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -149,15 +151,16 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -538,6 +541,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -855,7 +859,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1318,6 +1321,7 @@ k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 h1:tBEbstoM+K0FiBV5KGAKQ0 k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/unify-query/influxdb/client/client_test.go b/pkg/unify-query/influxdb/client/client_test.go index e1626cda9..f8df9d112 100644 --- a/pkg/unify-query/influxdb/client/client_test.go +++ b/pkg/unify-query/influxdb/client/client_test.go @@ -11,7 +11,6 @@ package client import ( "context" - "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -21,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) // TestDecodeResp diff --git a/pkg/unify-query/influxdb/decoder/json.go b/pkg/unify-query/influxdb/decoder/json.go index ec6f98971..182a419bb 100644 --- a/pkg/unify-query/influxdb/decoder/json.go +++ b/pkg/unify-query/influxdb/decoder/json.go @@ -12,9 +12,10 @@ package decoder import ( "bufio" "context" - "encoding/json" "fmt" "io" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) var jsonType = "application/json" diff --git a/pkg/unify-query/influxdb/influxdb_router.go b/pkg/unify-query/influxdb/influxdb_router.go index 7475f54fd..ee7a32231 100644 --- a/pkg/unify-query/influxdb/influxdb_router.go +++ b/pkg/unify-query/influxdb/influxdb_router.go @@ -12,7 +12,6 @@ package influxdb import ( "bytes" "context" - "encoding/json" "fmt" "math" "net/http" @@ -23,6 +22,7 @@ import ( "github.com/influxdata/influxdb/prometheus/remote" "google.golang.org/grpc" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/redis" diff --git a/pkg/unify-query/influxdb/influxdb_test.go b/pkg/unify-query/influxdb/influxdb_test.go index 5656d06e9..ced7a9dfb 100644 --- a/pkg/unify-query/influxdb/influxdb_test.go +++ b/pkg/unify-query/influxdb/influxdb_test.go @@ -11,7 +11,6 @@ package influxdb_test import ( "context" - "encoding/json" "fmt" "testing" "time" @@ -23,6 +22,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/mocktest" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) @@ -139,7 +139,7 @@ func TestGroupBy(t *testing.T) { result, err := json.Marshal(resultRows) assert.Nil(t, err) - assert.Equal(t, `[{"name":"result_0","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-125-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.1787488422133678,"2021-08-24T12:00:56Z"],[1.145359609368273,"2021-08-24T12:01:56Z"],[1.741398165950821,"2021-08-24T12:02:56Z"],[1.3780354592294548,"2021-08-24T12:03:56Z"],[1.3116959557365824,"2021-08-24T12:04:56Z"],[1.3860887098919377,"2021-08-24T12:05:56Z"],[1.4937898625358987,"2021-08-24T12:06:56Z"],[1.8885344972207012,"2021-08-24T12:07:56Z"],[1.543494673130278,"2021-08-24T12:08:56Z"],[1.6682035374979551,"2021-08-24T12:09:56Z"],[1.5992631668395854,"2021-08-24T12:10:56Z"],[1.6589861747802415,"2021-08-24T12:11:56Z"],[1.8569868081444194,"2021-08-24T12:12:56Z"]]},{"name":"result_1","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-139-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[0.5758157390226424,"2021-08-24T12:00:56Z"],[0.5424803872507733,"2021-08-24T12:01:56Z"],[1.0016694491164386,"2021-08-24T12:02:56Z"],[0.5257009345676369,"2021-08-24T12:03:56Z"],[0.5341791169361774,"2021-08-24T12:04:56Z"],[0.5175292153035469,"2021-08-24T12:05:56Z"],[0.5343129069539093,"2021-08-24T12:06:56Z"],[1.0183639399018447,"2021-08-24T12:07:56Z"],[0.6009013519658645,"2021-08-24T12:08:56Z"],[0.5424351164424456,"2021-08-24T12:09:56Z"],[0.534179116887594,"2021-08-24T12:10:56Z"],[0.5174428309400382,"2021-08-24T12:11:56Z"],[0.9931564010827747,"2021-08-24T12:12:56Z"]]},{"name":"result_2","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-21-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[9.189053479211022,"2021-08-24T12:00:56Z"],[8.276899924582034,"2021-08-24T12:01:56Z"],[12.835120640582524,"2021-08-24T12:02:56Z"],[8.560213797433004,"2021-08-24T12:03:56Z"],[8.341700135761048,"2021-08-24T12:04:56Z"],[9.03836488259614,"2021-08-24T12:05:56Z"],[7.750899808932389,"2021-08-24T12:06:56Z"],[12.286575158370558,"2021-08-24T12:07:56Z"],[21.77703268733308,"2021-08-24T12:08:56Z"],[19.20596365215852,"2021-08-24T12:09:56Z"],[7.947408092183444,"2021-08-24T12:10:56Z"],[8.202077746744619,"2021-08-24T12:11:56Z"],[12.780757625685412,"2021-08-24T12:12:56Z"]]},{"name":"result_3","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-38-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.6511007338849786,"2021-08-24T12:00:56Z"],[1.7019856498673305,"2021-08-24T12:01:56Z"],[2.5187656379220154,"2021-08-24T12:02:56Z"],[1.65165165150205,"2021-08-24T12:03:56Z"],[2.370617696044598,"2021-08-24T12:04:56Z"],[1.602403605341945,"2021-08-24T12:05:56Z"],[1.5179316096508726,"2021-08-24T12:06:56Z"],[2.219626168323257,"2021-08-24T12:07:56Z"],[1.5520694258432275,"2021-08-24T12:08:56Z"],[2.353923205358038,"2021-08-24T12:09:56Z"],[1.5502583764267808,"2021-08-24T12:10:56Z"],[1.5507753877137544,"2021-08-24T12:11:56Z"],[2.4028032704009368,"2021-08-24T12:12:56Z"]]},{"name":"result_5","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-56-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[30.82440300745231,"2021-08-24T12:00:56Z"],[22.06960479309041,"2021-08-24T12:01:56Z"],[46.45589842485261,"2021-08-24T12:02:56Z"],[22.392586352890703,"2021-08-24T12:03:56Z"],[22.24094355057531,"2021-08-24T12:04:56Z"],[32.44609164542279,"2021-08-24T12:05:56Z"],[24.9262909624078,"2021-08-24T12:06:56Z"],[40.18549746321112,"2021-08-24T12:07:56Z"],[23.38383838836149,"2021-08-24T12:08:56Z"],[25.06982649451891,"2021-08-24T12:09:56Z"],[32.27222832569239,"2021-08-24T12:10:56Z"],[20.83859560934145,"2021-08-24T12:11:56Z"]]},{"name":"result_6","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-7-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[38.62660944195749,"2021-08-24T12:00:56Z"],[25.379645943562686,"2021-08-24T12:01:56Z"],[29.377676997095005,"2021-08-24T12:02:56Z"],[26.59395973145055,"2021-08-24T12:03:56Z"],[25.579247817275082,"2021-08-24T12:04:56Z"],[27.90229161419864,"2021-08-24T12:05:56Z"],[59.69648696141153,"2021-08-24T12:06:56Z"],[52.722231557485564,"2021-08-24T12:07:56Z"],[23.719563392552622,"2021-08-24T12:08:56Z"],[37.785618279246805,"2021-08-24T12:09:56Z"],[34.96515240605265,"2021-08-24T12:10:56Z"],[51.99697935844213,"2021-08-24T12:11:56Z"]]},{"name":"result_7","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-86-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.2533422459144001,"2021-08-24T12:00:56Z"],[1.1363636363813223,"2021-08-24T12:01:56Z"],[1.2026056454229423,"2021-08-24T12:02:56Z"],[1.253132832399512,"2021-08-24T12:03:56Z"],[1.2525050101197799,"2021-08-24T12:04:56Z"],[1.2364243944267057,"2021-08-24T12:05:56Z"],[1.269417070427924,"2021-08-24T12:06:56Z"],[1.236217841627222,"2021-08-24T12:07:56Z"],[1.2199197861867666,"2021-08-24T12:08:56Z"],[1.2360113581358432,"2021-08-24T12:09:56Z"],[1.1693952558213456,"2021-08-24T12:10:56Z"],[1.1857047428511789,"2021-08-24T12:11:56Z"]]},{"name":"result_4","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"ip-10-0-1-48-n-bcs-k8s-40000","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[8.780160857742022,"2021-08-24T12:00:56Z"],[8.19369973196781,"2021-08-24T12:01:56Z"],[8.545576408043786,"2021-08-24T12:02:56Z"],[8.093163539039251,"2021-08-24T12:03:56Z"],[8.566638725556665,"2021-08-24T12:04:56Z"],[8.143431634897675,"2021-08-24T12:05:56Z"],[8.549874266729455,"2021-08-24T12:06:56Z"],[8.024794772875913,"2021-08-24T12:07:56Z"],[8.517773305818258,"2021-08-24T12:08:56Z"],[8.137583892570257,"2021-08-24T12:09:56Z"],[8.737212812628373,"2021-08-24T12:10:56Z"],[8.464632920074514,"2021-08-24T12:11:56Z"]]}]`, string(result)) + assert.JSONEq(t, `[{"name":"result_0","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-125-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.1787488422133678,"2021-08-24T12:00:56Z"],[1.145359609368273,"2021-08-24T12:01:56Z"],[1.741398165950821,"2021-08-24T12:02:56Z"],[1.3780354592294548,"2021-08-24T12:03:56Z"],[1.3116959557365824,"2021-08-24T12:04:56Z"],[1.3860887098919377,"2021-08-24T12:05:56Z"],[1.4937898625358987,"2021-08-24T12:06:56Z"],[1.8885344972207012,"2021-08-24T12:07:56Z"],[1.543494673130278,"2021-08-24T12:08:56Z"],[1.6682035374979551,"2021-08-24T12:09:56Z"],[1.5992631668395854,"2021-08-24T12:10:56Z"],[1.6589861747802415,"2021-08-24T12:11:56Z"],[1.8569868081444194,"2021-08-24T12:12:56Z"]]},{"name":"result_1","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-139-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[0.5758157390226424,"2021-08-24T12:00:56Z"],[0.5424803872507733,"2021-08-24T12:01:56Z"],[1.0016694491164386,"2021-08-24T12:02:56Z"],[0.5257009345676369,"2021-08-24T12:03:56Z"],[0.5341791169361774,"2021-08-24T12:04:56Z"],[0.5175292153035469,"2021-08-24T12:05:56Z"],[0.5343129069539093,"2021-08-24T12:06:56Z"],[1.0183639399018447,"2021-08-24T12:07:56Z"],[0.6009013519658645,"2021-08-24T12:08:56Z"],[0.5424351164424456,"2021-08-24T12:09:56Z"],[0.534179116887594,"2021-08-24T12:10:56Z"],[0.5174428309400382,"2021-08-24T12:11:56Z"],[0.9931564010827747,"2021-08-24T12:12:56Z"]]},{"name":"result_2","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-21-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[9.189053479211022,"2021-08-24T12:00:56Z"],[8.276899924582034,"2021-08-24T12:01:56Z"],[12.835120640582524,"2021-08-24T12:02:56Z"],[8.560213797433004,"2021-08-24T12:03:56Z"],[8.341700135761048,"2021-08-24T12:04:56Z"],[9.03836488259614,"2021-08-24T12:05:56Z"],[7.750899808932389,"2021-08-24T12:06:56Z"],[12.286575158370558,"2021-08-24T12:07:56Z"],[21.77703268733308,"2021-08-24T12:08:56Z"],[19.20596365215852,"2021-08-24T12:09:56Z"],[7.947408092183444,"2021-08-24T12:10:56Z"],[8.202077746744619,"2021-08-24T12:11:56Z"],[12.780757625685412,"2021-08-24T12:12:56Z"]]},{"name":"result_3","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-38-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.6511007338849786,"2021-08-24T12:00:56Z"],[1.7019856498673305,"2021-08-24T12:01:56Z"],[2.5187656379220154,"2021-08-24T12:02:56Z"],[1.65165165150205,"2021-08-24T12:03:56Z"],[2.370617696044598,"2021-08-24T12:04:56Z"],[1.602403605341945,"2021-08-24T12:05:56Z"],[1.5179316096508726,"2021-08-24T12:06:56Z"],[2.219626168323257,"2021-08-24T12:07:56Z"],[1.5520694258432275,"2021-08-24T12:08:56Z"],[2.353923205358038,"2021-08-24T12:09:56Z"],[1.5502583764267808,"2021-08-24T12:10:56Z"],[1.5507753877137544,"2021-08-24T12:11:56Z"],[2.4028032704009368,"2021-08-24T12:12:56Z"]]},{"name":"result_5","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-56-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[30.82440300745231,"2021-08-24T12:00:56Z"],[22.06960479309041,"2021-08-24T12:01:56Z"],[46.45589842485261,"2021-08-24T12:02:56Z"],[22.392586352890703,"2021-08-24T12:03:56Z"],[22.24094355057531,"2021-08-24T12:04:56Z"],[32.44609164542279,"2021-08-24T12:05:56Z"],[24.9262909624078,"2021-08-24T12:06:56Z"],[40.18549746321112,"2021-08-24T12:07:56Z"],[23.38383838836149,"2021-08-24T12:08:56Z"],[25.06982649451891,"2021-08-24T12:09:56Z"],[32.27222832569239,"2021-08-24T12:10:56Z"],[20.83859560934145,"2021-08-24T12:11:56Z"]]},{"name":"result_6","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-7-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[38.62660944195749,"2021-08-24T12:00:56Z"],[25.379645943562686,"2021-08-24T12:01:56Z"],[29.377676997095005,"2021-08-24T12:02:56Z"],[26.59395973145055,"2021-08-24T12:03:56Z"],[25.579247817275082,"2021-08-24T12:04:56Z"],[27.90229161419864,"2021-08-24T12:05:56Z"],[59.69648696141153,"2021-08-24T12:06:56Z"],[52.722231557485564,"2021-08-24T12:07:56Z"],[23.719563392552622,"2021-08-24T12:08:56Z"],[37.785618279246805,"2021-08-24T12:09:56Z"],[34.96515240605265,"2021-08-24T12:10:56Z"],[51.99697935844213,"2021-08-24T12:11:56Z"]]},{"name":"result_7","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"VM-1-86-centos","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[1.2533422459144001,"2021-08-24T12:00:56Z"],[1.1363636363813223,"2021-08-24T12:01:56Z"],[1.2026056454229423,"2021-08-24T12:02:56Z"],[1.253132832399512,"2021-08-24T12:03:56Z"],[1.2525050101197799,"2021-08-24T12:04:56Z"],[1.2364243944267057,"2021-08-24T12:05:56Z"],[1.269417070427924,"2021-08-24T12:06:56Z"],[1.236217841627222,"2021-08-24T12:07:56Z"],[1.2199197861867666,"2021-08-24T12:08:56Z"],[1.2360113581358432,"2021-08-24T12:09:56Z"],[1.1693952558213456,"2021-08-24T12:10:56Z"],[1.1857047428511789,"2021-08-24T12:11:56Z"]]},{"name":"result_4","tags":{"bk_biz_id":"2","bk_cloud_id":"0","bk_supplier_id":"0","bk_target_cloud_id":"0","bk_target_ip":"127.0.0.1","device_name":"cpu-total","hostname":"ip-10-0-1-48-n-bcs-k8s-40000","ip":"127.0.0.1"},"columns":["_value","_time"],"values":[[8.780160857742022,"2021-08-24T12:00:56Z"],[8.19369973196781,"2021-08-24T12:01:56Z"],[8.545576408043786,"2021-08-24T12:02:56Z"],[8.093163539039251,"2021-08-24T12:03:56Z"],[8.566638725556665,"2021-08-24T12:04:56Z"],[8.143431634897675,"2021-08-24T12:05:56Z"],[8.549874266729455,"2021-08-24T12:06:56Z"],[8.024794772875913,"2021-08-24T12:07:56Z"],[8.517773305818258,"2021-08-24T12:08:56Z"],[8.137583892570257,"2021-08-24T12:09:56Z"],[8.737212812628373,"2021-08-24T12:10:56Z"],[8.464632920074514,"2021-08-24T12:11:56Z"]]}]`, string(result)) } // TestMerge diff --git a/pkg/unify-query/influxdb/router_mock.go b/pkg/unify-query/influxdb/router_mock.go index ecec10e89..3ac53218b 100644 --- a/pkg/unify-query/influxdb/router_mock.go +++ b/pkg/unify-query/influxdb/router_mock.go @@ -287,7 +287,6 @@ func MockSpaceRouter(ctx context.Context) { func setSpaceTsDbMockData(ctx context.Context, bkAppSpace ir.BkAppSpace, spaceInfo ir.SpaceInfo, rtInfo ir.ResultTableDetailInfo, fieldInfo ir.FieldToResultTable, dataLabelInfo ir.DataLabelToResultTable) { mockRedisOnce.Do(func() { setRedisClient(ctx) - }) mockPath := "mock" + time.Now().String() diff --git a/pkg/unify-query/influxdb/spacetsdb_router_test.go b/pkg/unify-query/influxdb/spacetsdb_router_test.go index ebc1c3f61..970253e80 100644 --- a/pkg/unify-query/influxdb/spacetsdb_router_test.go +++ b/pkg/unify-query/influxdb/spacetsdb_router_test.go @@ -107,6 +107,8 @@ func (s *TestSuite) TestReloadByKey() { } space := router.GetSpace(s.ctx, "bkcc__2") + assert.NotNil(s.T(), space) + s.T().Logf("Space: %v\n", space) assert.Equal(s.T(), space["script_hhb_test.group3"].Filters[0]["bk_biz_id"], "2") diff --git a/pkg/unify-query/internal/json/json.go b/pkg/unify-query/internal/json/json.go new file mode 100644 index 000000000..44dca1f89 --- /dev/null +++ b/pkg/unify-query/internal/json/json.go @@ -0,0 +1,43 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package json + +import ( + "encoding/json" + "io" + + "github.com/bytedance/sonic" +) + +type Number struct { + json.Number +} + +var sonicAPI = sonic.Config{ + EscapeHTML: true, // 安全性 + CompactMarshaler: true, // 兼容性 + CopyString: true, // 正确性 +}.Froze() + +func Marshal(v interface{}) ([]byte, error) { + return sonicAPI.Marshal(v) +} + +func Unmarshal(data []byte, v interface{}) error { + return sonicAPI.Unmarshal(data, v) +} + +func NewEncoder(w io.Writer) sonic.Encoder { + return sonicAPI.NewEncoder(w) +} + +func NewDecoder(r io.Reader) sonic.Decoder { + return sonicAPI.NewDecoder(r) +} diff --git a/pkg/unify-query/metadata/queries.go b/pkg/unify-query/metadata/queries.go index 397ab148c..d27132e51 100644 --- a/pkg/unify-query/metadata/queries.go +++ b/pkg/unify-query/metadata/queries.go @@ -11,9 +11,10 @@ package metadata import ( "context" - "encoding/json" "github.com/prometheus/prometheus/model/labels" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) // SetQueries 写入查询扩展，因为有多个查询所以除了常量前缀之外，还需要指定 k（该值一般为指标名，用于对应） diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index d57d3ad7d..b7be78fcf 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -11,7 +11,6 @@ package metadata import ( "context" - "encoding/json" "fmt" "sort" "strings" @@ -20,6 +19,7 @@ import ( "github.com/VictoriaMetrics/metricsql" "github.com/prometheus/prometheus/model/labels" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" ) diff --git a/pkg/unify-query/mock/function.go b/pkg/unify-query/mock/function.go index 548d59aa1..a85f7e5df 100644 --- a/pkg/unify-query/mock/function.go +++ b/pkg/unify-query/mock/function.go @@ -10,7 +10,6 @@ package mock import ( - "encoding/json" "sort" "strings" @@ -18,6 +17,8 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) type TimeSeriesList []prompb.TimeSeries diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index de7338cb3..aef232a7e 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -11,7 +11,6 @@ package mock import ( "context" - "encoding/json" "fmt" "io" "net/http" @@ -19,6 +18,8 @@ import ( "sync" "github.com/jarcoal/httpmock" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) type VmRequest struct { diff --git a/pkg/unify-query/query/es/es_test.go b/pkg/unify-query/query/es/es_test.go index fb4c3e1c2..19f40a907 100644 --- a/pkg/unify-query/query/es/es_test.go +++ b/pkg/unify-query/query/es/es_test.go @@ -10,7 +10,6 @@ package es_test import ( - "encoding/json" "fmt" "testing" "time" @@ -22,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/es" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/es/mocktest" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" inner "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/es" ) diff --git a/pkg/unify-query/query/infos/infos.go b/pkg/unify-query/query/infos/infos.go index 497324ee2..a045db393 100644 --- a/pkg/unify-query/query/infos/infos.go +++ b/pkg/unify-query/query/infos/infos.go @@ -11,7 +11,6 @@ package infos import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -22,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" queryMod "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query" diff --git a/pkg/unify-query/query/promql/series.go b/pkg/unify-query/query/promql/series.go index 5fede62ec..2acd3abec 100644 --- a/pkg/unify-query/query/promql/series.go +++ b/pkg/unify-query/query/promql/series.go @@ -11,7 +11,6 @@ package promql import ( "context" - "encoding/json" "sort" "strconv" "time" @@ -22,6 +21,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 7f30f3570..81bf0c1dc 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -11,7 +11,6 @@ package structured import ( "context" - "encoding/json" "fmt" "math" "strings" @@ -24,6 +23,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" diff --git a/pkg/unify-query/query/structured/space_test.go b/pkg/unify-query/query/structured/space_test.go index a6cea12bf..df51a0373 100644 --- a/pkg/unify-query/query/structured/space_test.go +++ b/pkg/unify-query/query/structured/space_test.go @@ -11,13 +11,13 @@ package structured import ( "context" - "encoding/json" "sort" "testing" "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query" diff --git a/pkg/unify-query/query/structured/utils.go b/pkg/unify-query/query/structured/utils.go index 29ad43d71..7b54ad8fa 100644 --- a/pkg/unify-query/query/structured/utils.go +++ b/pkg/unify-query/query/structured/utils.go @@ -11,10 +11,10 @@ package structured import ( "context" - "encoding/json" "github.com/prometheus/prometheus/promql/parser" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query" ) diff --git a/pkg/unify-query/redis/router.go b/pkg/unify-query/redis/router.go index 7c2034114..e113fdebf 100644 --- a/pkg/unify-query/redis/router.go +++ b/pkg/unify-query/redis/router.go @@ -11,10 +11,11 @@ package redis import ( "context" - "encoding/json" "fmt" redis "github.com/go-redis/redis/v8" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) const ( diff --git a/pkg/unify-query/service/http/api.go b/pkg/unify-query/service/http/api.go index 66f835efb..f9e025a91 100644 --- a/pkg/unify-query/service/http/api.go +++ b/pkg/unify-query/service/http/api.go @@ -11,7 +11,6 @@ package http import ( "context" - "encoding/json" "fmt" "sort" "strconv" @@ -25,6 +24,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/infos" diff --git a/pkg/unify-query/service/http/api/api.go b/pkg/unify-query/service/http/api/api.go index 0d2ca6e19..a7a55bb7c 100644 --- a/pkg/unify-query/service/http/api/api.go +++ b/pkg/unify-query/service/http/api/api.go @@ -10,7 +10,6 @@ package api import ( - "encoding/json" "net/http" "time" @@ -18,6 +17,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/cmdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/cmdb/v1beta1" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" diff --git a/pkg/unify-query/service/http/check_handler.go b/pkg/unify-query/service/http/check_handler.go index 24c83c0b7..5acfc3dd8 100644 --- a/pkg/unify-query/service/http/check_handler.go +++ b/pkg/unify-query/service/http/check_handler.go @@ -11,7 +11,6 @@ package http import ( "context" - "encoding/json" "fmt" "net/http" "strings" @@ -20,6 +19,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/prometheus" diff --git a/pkg/unify-query/service/http/es.go b/pkg/unify-query/service/http/es.go index e293c1906..de60b969f 100644 --- a/pkg/unify-query/service/http/es.go +++ b/pkg/unify-query/service/http/es.go @@ -11,11 +11,11 @@ package http import ( "context" - "encoding/json" "io" "github.com/gin-gonic/gin" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" diff --git a/pkg/unify-query/service/http/es_test.go b/pkg/unify-query/service/http/es_test.go index eaa7b8f7f..a309e5277 100644 --- a/pkg/unify-query/service/http/es_test.go +++ b/pkg/unify-query/service/http/es_test.go @@ -10,7 +10,6 @@ package http_test import ( - "encoding/json" "fmt" "net/http/httptest" "strconv" @@ -25,6 +24,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/es" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/es/mocktest" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" servicehttp "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/service/http" ) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index 983de2fbc..edfb01327 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -10,13 +10,13 @@ package http import ( - "encoding/json" "fmt" "unsafe" "github.com/gin-gonic/gin" influxdbRouter "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" diff --git a/pkg/unify-query/service/http/handler_test.go b/pkg/unify-query/service/http/handler_test.go index 4d1548ea2..0a54109e0 100644 --- a/pkg/unify-query/service/http/handler_test.go +++ b/pkg/unify-query/service/http/handler_test.go @@ -13,7 +13,6 @@ import ( "bufio" "bytes" "context" - "encoding/json" "fmt" "net" "net/http" @@ -24,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/infos" diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index baab61784..ad8c4900b 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -11,7 +11,6 @@ package http import ( "context" - "encoding/json" "fmt" "sort" "strings" @@ -30,6 +29,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/downsample" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 443ec6ed7..56f375318 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -11,7 +11,6 @@ package http import ( "context" - "encoding/json" "fmt" "sort" "strconv" @@ -25,11 +24,14 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/featureFlag" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/promql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" + redisUtil "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/redis" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/redis" ) func TestQueryTsWithEs(t *testing.T) { @@ -2333,9 +2335,25 @@ func TestQueryTsClusterMetrics(t *testing.T) { mock.Init() promql.MockEngine() - influxdb.MockSpaceRouter(ctx) + var ( + key string + err error + ) + + key = fmt.Sprintf("%s:%s", ClusterMetricQueryPrefix, redis.ClusterMetricMetaKey) + _, err = redisUtil.HSet(ctx, key, "influxdb_shard_write_points_ok", `{"metric_name":"influxdb_shard_write_points_ok","tags":["bkm_cluster","database","engine","hostname","id","index_type","path","retention_policy","wal_path"]}`) + if err != nil { + return + } + + key = fmt.Sprintf("%s:%s", ClusterMetricQueryPrefix, redis.ClusterMetricKey) + _, err = redisUtil.HSet(ctx, key, "influxdb_shard_write_points_ok|bkm_cluster=default", `[{"bkm_cluster":"default","database":"_internal","engine":"tsm1","hostname":"influxdb-0","id":"43","index_type":"inmem","path":"/var/lib/influxdb/data/_internal/monitor/43","retention_policy":"monitor","wal_path":"/var/lib/influxdb/wal/_internal/monitor/43","time":1700903220,"value":1498687},{"bkm_cluster":"default","database":"_internal","engine":"tsm1","hostname":"influxdb-0","id":"44","index_type":"inmem","path":"/var/lib/influxdb/data/_internal/monitor/44","retention_policy":"monitor","wal_path":"/var/lib/influxdb/wal/_internal/monitor/44","time":1700903340,"value":1499039.5}]`) + if err != nil { + return + } + testCases := map[string]struct { query string result string @@ -2438,7 +2456,7 @@ func TestQueryTsClusterMetrics(t *testing.T) { "instant": true } `, - result: `{"series":[{"name":"_result0","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["bkm_cluster","database","engine","hostname","id","index_type","path","retention_policy","wal_path"],"group_values":["default","_internal","tsm1","influxdb-0","43","inmem","/var/lib/influxdb/data/_internal/monitor/43","monitor","/var/lib/influxdb/wal/_internal/monitor/43"],"values":[[1700903370000,0]]},{"name":"_result1","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["bkm_cluster","database","engine","hostname","id","index_type","path","retention_policy","wal_path"],"group_values":["default","_internal","tsm1","influxdb-0","44","inmem","/var/lib/influxdb/data/_internal/monitor/44","monitor","/var/lib/influxdb/wal/_internal/monitor/44"],"values":[[1700903370000,0]]}]}`, + result: `{"series":[{"name":"_result0","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["bkm_cluster","database","engine","hostname","id","index_type","path","retention_policy","wal_path"],"group_values":["default","_internal","tsm1","influxdb-0","43","inmem","/var/lib/influxdb/data/_internal/monitor/43","monitor","/var/lib/influxdb/wal/_internal/monitor/43"],"values":[[1700903220000,1498687]]},{"name":"_result1","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["bkm_cluster","database","engine","hostname","id","index_type","path","retention_policy","wal_path"],"group_values":["default","_internal","tsm1","influxdb-0","44","inmem","/var/lib/influxdb/data/_internal/monitor/44","monitor","/var/lib/influxdb/wal/_internal/monitor/44"],"values":[[1700903340000,1499039.5]]}]}`, }, } for name, c := range testCases { @@ -2468,6 +2486,7 @@ func TestQueryTsToInstanceAndStmt(t *testing.T) { mock.Init() promql.MockEngine() + influxdb.MockSpaceRouter(ctx) testCases := map[string]struct { query *structured.QueryTs diff --git a/pkg/unify-query/service/redis/redis.go b/pkg/unify-query/service/redis/redis.go index 86fc69a79..8be616ebf 100644 --- a/pkg/unify-query/service/redis/redis.go +++ b/pkg/unify-query/service/redis/redis.go @@ -71,7 +71,7 @@ func (s *Service) Reload(ctx context.Context) { out, err := redis.Ping(s.ctx) if err != nil { log.Errorf(context.TODO(), "redis ping errors: %s", err.Error()) - panic(err) + return } log.Warnf(context.TODO(), "redis service reloaded or start success, with %s", out) diff --git a/pkg/unify-query/tsdb/bksql/client.go b/pkg/unify-query/tsdb/bksql/client.go index d51a71a47..1562d18a5 100644 --- a/pkg/unify-query/tsdb/bksql/client.go +++ b/pkg/unify-query/tsdb/bksql/client.go @@ -11,13 +11,13 @@ package bksql import ( "context" - "encoding/json" "fmt" "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/bkapi" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" diff --git a/pkg/unify-query/tsdb/elasticsearch/agg_format.go b/pkg/unify-query/tsdb/elasticsearch/agg_format.go index 0499c3e3c..92a232c9c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/agg_format.go +++ b/pkg/unify-query/tsdb/elasticsearch/agg_format.go @@ -11,13 +11,14 @@ package elasticsearch import ( "context" - "encoding/json" "fmt" "strings" "sync" elastic "github.com/olivere/elastic/v7" "golang.org/x/exp/slices" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" ) var itemsPool = sync.Pool{ diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 97068a32b..ce27b93c1 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -11,7 +11,6 @@ package elasticsearch import ( "context" - "encoding/json" "errors" "fmt" "sort" @@ -24,6 +23,7 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" diff --git a/pkg/unify-query/tsdb/elasticsearch/format_test.go b/pkg/unify-query/tsdb/elasticsearch/format_test.go index bbe2885f8..53477c3d1 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/format_test.go @@ -11,7 +11,6 @@ package elasticsearch import ( "context" - "encoding/json" "testing" "time" @@ -19,6 +18,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" ) @@ -209,7 +209,7 @@ func TestFormatFactory_Query(t *testing.T) { body, _ := ss.Source() bodyJson, _ := json.Marshal(body) bodyString := string(bodyJson) - assert.Equal(t, c.expected, bodyString) + assert.JSONEq(t, c.expected, bodyString) }) } @@ -335,7 +335,7 @@ func TestFormatFactory_RangeQueryAndAggregates(t *testing.T) { body, _ := ss.Source() bodyJson, _ := json.Marshal(body) bodyString := string(bodyJson) - assert.Equal(t, c.expected, bodyString) + assert.JSONEq(t, c.expected, bodyString) }) } } @@ -383,7 +383,7 @@ func TestFormatFactory_AggDataFormat(t *testing.T) { outTs, err := json.Marshal(ts) assert.NoError(t, err) - assert.Equal(t, string(outTs), c.expected) + assert.JSONEq(t, string(outTs), c.expected) }) } } diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 04dccd7f3..8ef05e43c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -11,7 +11,6 @@ package elasticsearch import ( "context" - "encoding/json" "errors" "fmt" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index d83e4c67b..470e88714 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -11,7 +11,6 @@ package elasticsearch import ( "context" - "encoding/json" "fmt" "sync" "testing" @@ -21,6 +20,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" @@ -444,7 +444,7 @@ func TestInstance_queryReference(t *testing.T) { return } - assert.Equal(t, c.expected, timeSeries.String()) + assert.JSONEq(t, c.expected, timeSeries.String()) } else { var ( wg sync.WaitGroup @@ -469,7 +469,7 @@ func TestInstance_queryReference(t *testing.T) { assert.Equal(t, c.err, err) } else { res, _ := json.Marshal(list) - assert.Equal(t, c.expected, string(res)) + assert.JSONEq(t, c.expected, string(res)) } } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 8a21be654..cc790c688 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -11,12 +11,12 @@ package elasticsearch import ( "context" - "encoding/json" "fmt" "testing" "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" ) @@ -83,7 +83,7 @@ func TestQsToDsl(t *testing.T) { body, _ := query.Source() bodyJson, _ := json.Marshal(body) bodyString := string(bodyJson) - assert.Equal(t, c.expected, bodyString) + assert.JSONEq(t, c.expected, bodyString) } } else { assert.Equal(t, c.err, err) diff --git a/pkg/unify-query/tsdb/influxdb/instance.go b/pkg/unify-query/tsdb/influxdb/instance.go index 6efea9b07..e19d63505 100644 --- a/pkg/unify-query/tsdb/influxdb/instance.go +++ b/pkg/unify-query/tsdb/influxdb/instance.go @@ -11,7 +11,6 @@ package influxdb import ( "context" - "encoding/json" "fmt" "io" "net/url" @@ -35,6 +34,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" diff --git a/pkg/unify-query/tsdb/offlineDataArchive/instance.go b/pkg/unify-query/tsdb/offlineDataArchive/instance.go index 38e013860..41b313fc7 100644 --- a/pkg/unify-query/tsdb/offlineDataArchive/instance.go +++ b/pkg/unify-query/tsdb/offlineDataArchive/instance.go @@ -11,7 +11,6 @@ package offlineDataArchive import ( "context" - "encoding/json" "fmt" "sync" "time" @@ -28,6 +27,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" influxdbRouter "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" diff --git a/pkg/unify-query/tsdb/redis/instance.go b/pkg/unify-query/tsdb/redis/instance.go index 7da9d7a79..882809faa 100644 --- a/pkg/unify-query/tsdb/redis/instance.go +++ b/pkg/unify-query/tsdb/redis/instance.go @@ -11,7 +11,6 @@ package redis import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -26,6 +25,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metric" diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance.go b/pkg/unify-query/tsdb/victoriaMetrics/instance.go index d4702f1bc..f92e9918f 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance.go @@ -11,7 +11,6 @@ package victoriaMetrics import ( "context" - "encoding/json" "fmt" "sort" "strings" @@ -26,6 +25,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go index 2c6b9cf60..61da29e76 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go @@ -11,7 +11,6 @@ package victoriaMetrics import ( "context" - "encoding/json" "fmt" "sort" "testing" @@ -21,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/curl" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" From 70c8b37a16b791459ac95b13d7e33072556d038e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Mar 2025 09:51:36 +0800 Subject: [PATCH 029/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 3e52ab0f3..c092fc070 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -137,7 +137,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, }, }, - want: "CAST(`object`[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", + want: "CAST(object[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", }, { name: "doris t8est text field wildcard", @@ -151,7 +151,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, }, }, - want: "CAST(`object`[\"field\"] AS STRING) LIKE '%partial%'", + want: "CAST(object[\"field\"] AS STRING) LIKE '%partial%'", }, { name: "doris test OR condition", From 03d1a8720705626052ba3711cba12cdc055ea7fd Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Mar 2025 11:35:00 +0800 Subject: [PATCH 030/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format_test.go | 57 +++++++++++++++++---- pkg/unify-query/tsdb/bksql/instance_test.go | 6 +-- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 11 +++- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index f3544a3d3..04b6d37e0 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -22,8 +22,8 @@ import ( ) func TestNewSqlFactory(t *testing.T) { - start := time.Unix(1717144141, 0) - end := time.Unix(1717147741, 0) + start := time.Unix(1741795260, 0) + end := time.Unix(1741796260, 0) for name, c := range map[string]struct { query *metadata.Query @@ -34,14 +34,14 @@ func TestNewSqlFactory(t *testing.T) { }{ "doris sum-count_over_time-with-promql-1": { query: &metadata.Query{ - DB: "100133_ieod_logsearch4_errorlog_p", + DB: "5000140_bklog_container_log_demo_analysis", Measurement: "doris", Field: "gseIndex", Aggregates: metadata.Aggregates{ { Name: "count", Dimensions: []string{ - "ip", + "level", }, Window: time.Minute, }, @@ -53,13 +53,52 @@ func TestNewSqlFactory(t *testing.T) { Operator: metadata.ConditionGt, Value: []string{"0"}, }, + { + DimensionName: "level", + Operator: metadata.ConditionEqual, + Value: []string{"ERROR"}, + }, + }, + }, + From: 0, + Size: 0, + Orders: metadata.Orders{"level": true}, + }, + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + }, + "doris sum-count_over_time-with-promql-seconds": { + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{ + "level", + }, + Window: time.Minute + time.Second*15, + }, + }, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "gseIndex", + Operator: metadata.ConditionGt, + Value: []string{"0"}, + }, + { + DimensionName: "level", + Operator: metadata.ConditionEqual, + Value: []string{"ERROR"}, + }, }, }, From: 0, Size: 0, - Orders: metadata.Orders{"ip": true}, + Orders: metadata.Orders{"level": true}, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC, `ip` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 1000 / 75 AS INT) * 75 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-with-promql-1": { query: &metadata.Query{ @@ -87,7 +126,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 10, Orders: nil, }, - expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", + expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, "doris count-with-count-promql-1": { query: &metadata.Query{ @@ -104,7 +143,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1717144141000 AND `dtEventTimeStamp` < 1717147741000 AND `thedate` = '20240531' GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -136,7 +175,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 7f7731cf6..e5d820f55 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -507,7 +507,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, __shard_key__ * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), __shard_key__ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例11: doris 处理 object 字段 + 时间聚合 5m { @@ -531,7 +531,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(__shard_key__ / 1000 / 5 AS INT) * 5 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例12: doris 处理 object 字段 + 时间聚合 15s { @@ -555,7 +555,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 1000 / 15 AS INT) * 15 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 9d058889b..02f762c57 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -92,9 +92,16 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { - groupByFields = append(groupByFields, ShardKey) - selectFields = append(selectFields, fmt.Sprintf("%s * 60 AS `%s`", ShardKey, TimeStamp)) + // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 + var timeField string + if int64(agg.Window.Seconds())%60 == 0 { + timeField = fmt.Sprintf(`CAST(%s / 1000 / %.f AS INT) * %.f * 60 * 1000`, ShardKey, agg.Window.Minutes(), agg.Window.Minutes()) + } else { + timeField = fmt.Sprintf(`CAST(%s / 1000 / %.f AS INT) * %.f * 1000`, d.timeField, agg.Window.Seconds(), agg.Window.Seconds()) + } + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", timeField, TimeStamp)) + groupByFields = append(groupByFields, TimeStamp) orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) } } From 1bb0adaf4157169d468de6c76b9147077a435ed7 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Mar 2025 14:22:10 +0800 Subject: [PATCH 031/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 02f762c57..3d31c6f00 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -191,6 +191,28 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) switch c.Operator { // 处理等于类操作符（=, IN） case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) > 1 && !c.IsWildcard { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + if c.IsWildcard { + op = "LIKE" + val = fmt.Sprintf("'%%%s%%'", c.Value[0]) + } else { + if d.checkMatchALL(c.DimensionName) { + op = "MATCH_PHRASE_PREFIX" + } else { + op = "=" + } + } + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, v)) + } + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) + } + if len(c.Value) == 1 { if c.IsWildcard { op = "LIKE" @@ -317,7 +339,7 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { field, _ := d.dimTransform(c.Field) if d.checkMatchALL(c.Field) { - return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil + return fmt.Sprintf("%s MATCH_PHRASE_PREFIX '%%%s%%'", field, c.Value), nil } return fmt.Sprintf("%s = '%s'", field, c.Value), nil From 16b09b84f56414c10a23f53e507d0efdbf173d9b Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 13 Mar 2025 22:04:39 +0800 Subject: [PATCH 032/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 7 +- pkg/unify-query/tsdb/bksql/instance_test.go | 137 +++++++++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 58 ++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 64 +++++--- .../tsdb/bksql/sqlExpr/doris_test.go | 2 +- 5 files changed, 226 insertions(+), 42 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 1ac88a9e4..eda194b6c 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -84,14 +84,17 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f.timeField = dtEventTimeStamp } - fieldsMap := make(map[string]string) f.expr = sqlExpr.GetSQLExpr(f.query.Measurement). - WithFieldsMap(fieldsMap). WithInternalFields(f.timeField, query.Field). WithEncode(metadata.GetPromDataFormat(ctx).EncodeFunc()) return f } +func (f *QueryFactory) WithFieldsMap(fieldsMap map[string]string) *QueryFactory { + f.expr.WithFieldsMap(fieldsMap) + return f +} + func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { f.start = start f.end = end diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index e5d820f55..54ad9bf73 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -173,6 +173,131 @@ func TestInstance_bkSql(t *testing.T) { }, expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", }, + { + query: &metadata.Query{ + DB: "132_lol_new_login_queue_login_1min", + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, + }, + }, + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test')", + }, + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test')", + }, + { + query: &metadata.Query{ + DB: "132_lol_new_login_queue_login_1min", + Measurement: sqlExpr.Doris, + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "text", + Operator: metadata.ConditionNotEqual, + Value: []string{"test"}, + }, + }, + }, + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test') AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND (`text` NOT LIKE '%test%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test')", + }, + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test') AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND (`text` NOT LIKE '%test%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test')", + }, + { + query: &metadata.Query{ + DB: "132_lol_new_login_queue_login_1min", + Field: "login_rate", + AllConditions: metadata.AllConditions{ + []metadata.ConditionField{ + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test", "test2"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test", "test2"}, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test"}, + IsWildcard: true, + }, + { + DimensionName: "namespace", + Operator: metadata.ConditionContains, + Value: []string{"test"}, + }, + }, + }, + BkSqlCondition: "(`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND (`namespace` LIKE '%test%') AND (`namespace` = 'test')", + }, + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND (`namespace` LIKE '%test%') AND (`namespace` = 'test')", + }, { query: &metadata.Query{ DB: "132_hander_opmon_avg", @@ -267,13 +392,17 @@ func TestInstance_bkSql(t *testing.T) { c.end = end } - condition, err := sqlExpr.GetSQLExpr(c.query.Measurement).WithFieldsMap(nil).ParserAllConditions(c.query.AllConditions) + fieldsMap := map[string]string{ + "text": sqlExpr.DorisTypeText, + } + + condition, err := sqlExpr.GetSQLExpr(c.query.Measurement).WithFieldsMap(fieldsMap).ParserAllConditions(c.query.AllConditions) assert.Nil(t, err) if err == nil { assert.Equal(t, c.query.BkSqlCondition, condition) } - fact := bksql.NewQueryFactory(ctx, c.query).WithRangeTime(c.start, c.end) + fact := bksql.NewQueryFactory(ctx, c.query).WithFieldsMap(fieldsMap).WithRangeTime(c.start, c.end) sql, err := fact.SQL() assert.Nil(t, err) assert.Equal(t, c.expected, sql) @@ -574,7 +703,9 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { } // SQL生成验证 - fact := bksql.NewQueryFactory(ctx, tc.query).WithRangeTime(start, end) + fact := bksql.NewQueryFactory(ctx, tc.query).WithFieldsMap(map[string]string{ + "text": sqlExpr.DorisTypeText, + }).WithRangeTime(start, end) generatedSQL, err := fact.SQL() if tc.err != nil { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 674af9c02..cc97755a2 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -275,28 +275,57 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro // 根据操作符类型生成不同的SQL表达式 switch c.Operator { - // 处理等于类操作符（=, IN） + // 处理等于类操作符（=, IN, LIKE） case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: - if len(c.Value) == 1 { + if len(c.Value) > 1 && !c.IsWildcard { + op = "IN" + val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + var format string if c.IsWildcard { + format = "'%%%s%%'" op = "LIKE" - val = fmt.Sprintf("'%%%s%%'", c.Value[0]) } else { + format = "'%s'" op = "=" - val = fmt.Sprintf("'%s'", c.Value[0]) } - } else { - op = "IN" - val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) + } } - // 处理不等于类操作符（!=, NOT IN） + // 处理不等于类操作符（!=, NOT IN, NOT LIKE） case metadata.ConditionNotEqual, metadata.ConditionNotContains: - if len(c.Value) == 1 { - op = "!=" - val = fmt.Sprintf("'%s'", c.Value[0]) - } else { + if len(c.Value) > 1 && !c.IsWildcard { op = "NOT IN" val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "NOT LIKE" + } else { + format = "'%s'" + op = "!=" + } + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) + } } // 处理正则表达式匹配 case metadata.ConditionRegEqual: @@ -334,5 +363,8 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro return "", fmt.Errorf("unknown operator %s", c.Operator) } - return fmt.Sprintf("%s %s %s", key, op, val), nil + if key != "" { + return fmt.Sprintf("%s %s %s", key, op, val), nil + } + return val, nil } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 3d31c6f00..de8c2a0e2 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -189,16 +189,18 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) // 根据操作符类型生成不同的SQL表达式 switch c.Operator { - // 处理等于类操作符（=, IN） + // 处理等于类操作符（=, IN, LIKE） case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: - if len(c.Value) > 1 && !c.IsWildcard { + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { op = "IN" val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) } else { + var format string if c.IsWildcard { + format = "'%%%s%%'" op = "LIKE" - val = fmt.Sprintf("'%%%s%%'", c.Value[0]) } else { + format = "'%s'" if d.checkMatchALL(c.DimensionName) { op = "MATCH_PHRASE_PREFIX" } else { @@ -208,31 +210,44 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) var filter []string for _, v := range c.Value { - filter = append(filter, fmt.Sprintf("%s %s %s", key, op, v)) + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) } - val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) - } - - if len(c.Value) == 1 { - if c.IsWildcard { - op = "LIKE" - val = fmt.Sprintf("'%%%s%%'", c.Value[0]) + key = "" + if len(filter) == 1 { + val = filter[0] } else { - op = "=" - val = fmt.Sprintf("'%s'", c.Value[0]) + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) } - } else { - op = "IN" - val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) } - // 处理不等于类操作符（!=, NOT IN） + // 处理不等于类操作符（!=, NOT IN, NOT LIKE） case metadata.ConditionNotEqual, metadata.ConditionNotContains: - if len(c.Value) == 1 { - op = "!=" - val = fmt.Sprintf("'%s'", c.Value[0]) - } else { + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { op = "NOT IN" val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + } else { + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "NOT LIKE" + } else { + format = "'%s'" + if d.checkMatchALL(c.DimensionName) { + op = "NOT MATCH_PHRASE_PREFIX" + } else { + op = "!=" + } + } + + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) + } } // 处理正则表达式匹配 case metadata.ConditionRegEqual: @@ -270,7 +285,10 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) return "", fmt.Errorf("unknown operator %s", c.Operator) } - return fmt.Sprintf("%s %s %s", key, op, val), nil + if key != "" { + return fmt.Sprintf("%s %s %s", key, op, val), nil + } + return val, nil } func (d *DorisSQLExpr) checkMatchALL(k string) bool { @@ -339,7 +357,7 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { field, _ := d.dimTransform(c.Field) if d.checkMatchALL(c.Field) { - return fmt.Sprintf("%s MATCH_PHRASE_PREFIX '%%%s%%'", field, c.Value), nil + return fmt.Sprintf("%s MATCH_PHRASE_PREFIX '%s'", field, c.Value), nil } return fmt.Sprintf("%s = '%s'", field, c.Value), nil diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index c092fc070..c63f1b8ed 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -84,7 +84,7 @@ func TestDorisSQLExpr_ParserQueryString(t *testing.T) { { name: "text filter", input: "text:value", - want: "`text` LIKE '%value%'", + want: "`text` MATCH_PHRASE_PREFIX 'value'", }, { name: "object field", From f025797ac1c3bac13613089d3d4783886cb334ad Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 17 Mar 2025 18:10:41 +0800 Subject: [PATCH 033/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/method.go | 5 +++-- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 12 +++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 10 ++++++++++ pkg/unify-query/tsdb/prometheus/querier.go | 10 ---------- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index 93aa652ff..915fb5714 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/promql/parser" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" @@ -54,9 +55,9 @@ func (a AggregateMethodList) ToQry(timezone string) metadata.Aggregates { } if aggr.Window != "" { - window, err := time.ParseDuration(string(aggr.Window)) + window, err := model.ParseDuration(string(aggr.Window)) if err == nil { - agg.Window = window + agg.Window = time.Duration(window) } } aggs = append(aggs, agg) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index cc97755a2..169ebc150 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -13,6 +13,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" ) @@ -165,7 +166,16 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { - timeField := fmt.Sprintf("(`%s` - (`%s` %% %d))", d.timeField, d.timeField, agg.Window.Milliseconds()) + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(agg.TimeZone) + if locErr != nil { + loc = time.UTC + } + // 获取时区偏移量 + _, offset := time.Now().In(loc).Zone() + offsetMillis := offset * 1000 + + timeField := fmt.Sprintf("(`%s` - ((`%s` - %d) %% %d - %d))", d.timeField, d.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) groupByFields = append(groupByFields, timeField) selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index de8c2a0e2..337d9d157 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -12,6 +12,7 @@ package sqlExpr import ( "fmt" "strings" + "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" @@ -92,6 +93,15 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(agg.TimeZone) + if locErr != nil { + loc = time.UTC + } + // 获取时区偏移量 + _, offset := time.Now().In(loc).Zone() + fmt.Println(offset) + // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 var timeField string if int64(agg.Window.Seconds())%60 == 0 { diff --git a/pkg/unify-query/tsdb/prometheus/querier.go b/pkg/unify-query/tsdb/prometheus/querier.go index 69f0f8825..d27ef32c0 100644 --- a/pkg/unify-query/tsdb/prometheus/querier.go +++ b/pkg/unify-query/tsdb/prometheus/querier.go @@ -164,16 +164,6 @@ func (q *Querier) selectFn(hints *storage.SelectHints, matchers ...*labels.Match // 获取因转毫秒丢失的时间精度 startTime = function.MsIntMergeNs(hints.Start, qp.Start) endTime = function.MsIntMergeNs(hints.End, qp.End) - - if len(query.qry.Aggregates) == 1 { - agg := query.qry.Aggregates[0] - - // 如果使用时间聚合计算，是否对齐开始时间 - if agg.Window.Milliseconds() > 0 { - ns := intMathFloor(startTime.UnixNano(), agg.Window.Nanoseconds()) * agg.Window.Nanoseconds() - startTime = time.Unix(0, ns) - } - } } setCh <- query.instance.QuerySeriesSet(ctx, query.qry, startTime, endTime) From c26960653a028cf7c8922054e1fdeaaf28c65007 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 19 Mar 2025 09:59:08 +0800 Subject: [PATCH 034/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../query/structured/query_ts_test.go | 68 +++++++++++-------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index a12d10107..39adc1174 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -1196,53 +1196,63 @@ func TestQueryTs_ToQueryReference(t *testing.T) { func TestTimeOffset(t *testing.T) { for name, c := range map[string]struct { - t time.Time - tz string - step time.Duration - actual time.Time + t time.Time + tz string + step time.Duration + expected time.Time + + out string }{ "test align": { - t: time.Unix(1701306000, 0), // 2023-11-30 09:00:00 +0800 ~ 2024-05-30 09:00:00 +0800 - tz: "Asia/Shanghai", - step: time.Hour * 3, + t: time.Unix(1701306000, 0), + tz: "Asia/Shanghai", + step: time.Hour * 3, + expected: time.UnixMilli(1701306000000), + out: `2023-11-30 09:00:00 +0800 CST => 2023-11-30 09:00:00 +0800 CST`, }, "test align -1": { - t: time.Unix(1703732400, 0), // 2023-11-30 09:00:00 +0800 ~ 2024-05-30 09:00:00 +0800 - tz: "Asia/Shanghai", - step: time.Hour * 3, + t: time.Unix(1703732400, 0), + tz: "Asia/Shanghai", + step: time.Hour * 3, + expected: time.UnixMilli(1703725200000), + out: `2023-12-28 11:00:00 +0800 CST => 2023-12-28 09:00:00 +0800 CST`, }, "test align - 2": { - t: time.Unix(1730082578, 0), // 2024-10-28 10:29:38 +0800 ~ 2024-10-28 10:12:00 +0800 - tz: "Asia/Shanghai", - step: time.Minute * 18, + t: time.Unix(1730082578, 0), + tz: "Asia/Shanghai", + step: time.Minute * 18, + expected: time.UnixMilli(1730081520000), + out: `2024-10-28 10:29:38 +0800 CST => 2024-10-28 10:12:00 +0800 CST`, }, "test align - 3": { - t: time.Unix(1741190400, 0), // 2024-10-28 10:29:38 +0800 ~ 2024-10-28 10:12:00 +0800 - tz: "Asia/Shanghai", - step: time.Hour * 24, + t: time.Unix(1741190400, 0), + tz: "Asia/Shanghai", + step: time.Hour * 24, + expected: time.UnixMilli(1741190400000), + out: `2025-03-06 00:00:00 +0800 CST => 2025-03-06 00:00:00 +0800 CST`, }, "test alian - 4": { - t: time.UnixMilli(1741336672161), - tz: "Asia/Shanghai", - step: time.Hour * 24, - actual: time.UnixMilli(1741276800000), + t: time.UnixMilli(1741336672161), + tz: "Asia/Shanghai", + step: time.Hour * 24, + expected: time.UnixMilli(1741276800000), + out: `2025-03-07 16:37:52.161 +0800 CST => 2025-03-07 00:00:00 +0800 CST`, }, "test alian - 5": { - t: time.UnixMilli(1741336672161), - tz: "UTC", - step: time.Hour * 24, - actual: time.UnixMilli(1741305600000), + t: time.UnixMilli(1741336672161), + tz: "UTC", + step: time.Hour * 24, + expected: time.UnixMilli(1741305600000), + out: `2025-03-07 16:37:52.161 +0800 CST => 2025-03-07 00:00:00 +0000 UTC`, }, } { t.Run(name, func(t *testing.T) { tz1, t1, err := timeOffset(c.t, c.tz, c.step) assert.Nil(t, err) - fmt.Println(c.tz, "=>", tz1) - fmt.Println(c.t.String(), "=>", t1.String()) - fmt.Println(c.t, "=>", t1.Unix()) - - assert.Equal(t, c.actual.UnixMilli(), t1.UnixMilli()) + assert.Equal(t, c.tz, tz1) + assert.Equal(t, c.out, fmt.Sprintf("%s => %s", c.t.String(), t1.String())) + assert.Equal(t, c.expected.UnixMilli(), t1.UnixMilli()) }) } } From 94da0455456c50cb07deda9e230853d1393b68f1 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 20 Mar 2025 15:51:24 +0800 Subject: [PATCH 035/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../query/structured/query_ts_test.go | 10 ++- pkg/unify-query/service/http/query.go | 8 +-- pkg/unify-query/tsdb/bksql/format_test.go | 2 +- pkg/unify-query/tsdb/bksql/instance_test.go | 24 ++++---- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 61 +++++++++++-------- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 32 +++++++--- .../tsdb/bksql/sqlExpr/doris_test.go | 4 +- 7 files changed, 85 insertions(+), 56 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index 2365062bb..6c9e2c43b 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -11,6 +11,7 @@ package structured import ( "context" + "fmt" "testing" "time" @@ -18,6 +19,8 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" md "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/promql" @@ -270,7 +273,9 @@ func TestQueryToMetric(t *testing.T) { metric, err := c.query.ToQueryMetric(ctx, spaceUID) assert.Nil(t, err) - assert.Equal(t, c.metric, metric) + actual, _ := json.Marshal(metric) + expect, _ := json.Marshal(c.metric) + assert.JSONEq(t, string(expect), string(actual)) }) } } @@ -1246,9 +1251,8 @@ func TestTimeOffset(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - tz1, t1, err := timeOffset(c.t, c.tz, c.step) + tz1, t1 := function.TimeOffset(c.t, c.tz, c.step) - assert.Nil(t, err) assert.Equal(t, c.tz, tz1) assert.Equal(t, c.out, fmt.Sprintf("%s => %s", c.t.String(), t1.String())) assert.Equal(t, c.expected.UnixMilli(), t1.UnixMilli()) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 268603d75..629f09903 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -334,7 +334,7 @@ func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.Query } queryRef, err := queryTs.ToQueryReference(ctx) - unit, startTime, endTime, err := function.QueryTimestamp(query.Start, query.End) + unit, startTime, endTime, err := function.QueryTimestamp(queryTs.Start, queryTs.End) if err != nil { log.Errorf(ctx, err.Error()) return nil, err @@ -364,7 +364,7 @@ func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.Query // 只有聚合场景需要对齐 if window, windowErr := queryTs.GetMaxWindow(); windowErr == nil && window.Seconds() > 0 { - startTime, endTime, step, _, err = structured.AlignTime(startTime, endTime, query.Step, query.Timezone) + startTime, endTime, step, _, err = structured.AlignTime(startTime, endTime, queryTs.Step, queryTs.Timezone) if err != nil { return nil, err } @@ -373,9 +373,9 @@ func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.Query } if queryTs.Instant { - res, err = instance.DirectQuery(ctx, query.MetricMerge, startTime) + res, err = instance.DirectQuery(ctx, queryTs.MetricMerge, startTime) } else { - res, err = instance.DirectQueryRange(ctx, query.MetricMerge, startTime, endTime, step) + res, err = instance.DirectQueryRange(ctx, queryTs.MetricMerge, startTime, endTime, step) } if err != nil { return nil, err diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 04b6d37e0..717cb9e19 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -98,7 +98,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 0, Orders: metadata.Orders{"level": true}, }, - expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 1000 / 75 AS INT) * 75 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 75000 AS INT) * 75000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-with-promql-1": { query: &metadata.Query{ diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 54ad9bf73..79939a597 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -32,9 +32,9 @@ func TestInstance_QueryRaw(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", - "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", - "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - (`dtEventTimeStamp` % 60000)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) end := time.UnixMilli(1730118889181) @@ -171,7 +171,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - (`dtEventTimeStamp` % 15000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0)) ORDER BY `_timestamp_` ASC", }, { query: &metadata.Query{ @@ -203,9 +203,9 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test')", + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test')", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, { query: &metadata.Query{ @@ -260,9 +260,9 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test') AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND (`text` NOT LIKE '%test%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test')", + BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND (`namespace` NOT LIKE '%test%') AND (`namespace` != 'test') AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND (`text` NOT LIKE '%test%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test')", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, { query: &metadata.Query{ @@ -294,9 +294,9 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - BkSqlCondition: "(`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND (`namespace` LIKE '%test%') AND (`namespace` = 'test')", + BkSqlCondition: "(`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND (`namespace` LIKE '%test%') AND (`namespace` = 'test')", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, { query: &metadata.Query{ @@ -378,7 +378,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - (`dtEventTimeStamp` % 3600000))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - (`dtEventTimeStamp` % 3600000)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0)) ORDER BY `_timestamp_` ASC", }, } @@ -684,7 +684,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 1000 / 15 AS INT) * 15 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 169ebc150..5af5d75c2 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -126,18 +126,6 @@ func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { return d } -func (d *DefaultSQLExpr) dimTransform(s string) (string, error) { - if s == "" { - return "", nil - } - fs := strings.Split(s, ".") - if len(fs) > 1 { - return "", fmt.Errorf("query is not support object with %s", s) - } - - return fmt.Sprintf("`%s`", s), nil -} - // ParserQueryString 解析查询字符串（当前实现返回空） func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { return "", nil @@ -166,14 +154,18 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { - // 时间聚合函数兼容时区 - loc, locErr := time.LoadLocation(agg.TimeZone) - if locErr != nil { - loc = time.UTC + + var offsetMillis int + if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(agg.TimeZone) + if locErr != nil { + loc = time.UTC + } + // 获取时区偏移量 + _, offset := time.Now().In(loc).Zone() + offsetMillis = offset * 1e3 } - // 获取时区偏移量 - _, offset := time.Now().In(loc).Zone() - offsetMillis := offset * 1000 timeField := fmt.Sprintf("(`%s` - ((`%s` - %d) %% %d - %d))", d.timeField, d.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) @@ -260,13 +252,6 @@ func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllCondition } // buildCondition 构建单个条件表达式 -// 参数： -// -// c - 条件字段对象 -// -// 返回值： -// -// 生成的SQL条件表达式字符串和错误信息 func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { if len(c.Value) == 0 { return "", nil @@ -283,6 +268,11 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro return "", err } + // 对值进行转义处理 + for i, v := range c.Value { + c.Value[i] = d.valueTransform(v) + } + // 根据操作符类型生成不同的SQL表达式 switch c.Operator { // 处理等于类操作符（=, IN, LIKE） @@ -378,3 +368,22 @@ func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, erro } return val, nil } + +func (d *DefaultSQLExpr) valueTransform(s string) string { + if strings.Contains(s, "'") { + s = strings.ReplaceAll(s, "'", "''") + } + return s +} + +func (d *DefaultSQLExpr) dimTransform(s string) (string, error) { + if s == "" { + return "", nil + } + fs := strings.Split(s, ".") + if len(fs) > 1 { + return "", fmt.Errorf("query is not support object with %s", s) + } + + return fmt.Sprintf("`%s`", s), nil +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 337d9d157..4ece72a6e 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -93,21 +93,25 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) if agg.Window > 0 { - // 时间聚合函数兼容时区 - loc, locErr := time.LoadLocation(agg.TimeZone) - if locErr != nil { - loc = time.UTC - } + // 获取时区偏移量 - _, offset := time.Now().In(loc).Zone() - fmt.Println(offset) + var offset int + if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(agg.TimeZone) + if locErr != nil { + loc = time.UTC + } + _, offset = time.Now().In(loc).Zone() + fmt.Println(offset) + } // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 var timeField string if int64(agg.Window.Seconds())%60 == 0 { timeField = fmt.Sprintf(`CAST(%s / 1000 / %.f AS INT) * %.f * 60 * 1000`, ShardKey, agg.Window.Minutes(), agg.Window.Minutes()) } else { - timeField = fmt.Sprintf(`CAST(%s / 1000 / %.f AS INT) * %.f * 1000`, d.timeField, agg.Window.Seconds(), agg.Window.Seconds()) + timeField = fmt.Sprintf(`CAST(%s / %d AS INT) * %d `, d.timeField, agg.Window.Milliseconds(), agg.Window.Milliseconds()) } selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", timeField, TimeStamp)) @@ -197,6 +201,11 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) key, _ = d.dimTransform(c.DimensionName) + // 对值进行转义处理 + for i, v := range c.Value { + c.Value[i] = d.valueTransform(v) + } + // 根据操作符类型生成不同的SQL表达式 switch c.Operator { // 处理等于类操作符（=, IN, LIKE） @@ -419,6 +428,13 @@ func (d *DorisSQLExpr) dimTransform(s string) (string, bool) { return fmt.Sprintf("`%s`", s), false } +func (d *DorisSQLExpr) valueTransform(s string) string { + if strings.Contains(s, "'") { + s = strings.ReplaceAll(s, "'", "''") + } + return s +} + func init() { Register(Doris, &DorisSQLExpr{}) } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index c63f1b8ed..2c59428af 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -127,7 +127,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { { { DimensionName: "object.field", - Value: []string{"test-value"}, + Value: []string{"What's UP"}, Operator: metadata.ConditionEqual, }, { @@ -137,7 +137,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, }, }, - want: "CAST(object[\"field\"] AS STRING) = 'test-value' AND `tag` != 'test'", + want: "CAST(object[\"field\"] AS STRING) MATCH_PHRASE_PREFIX 'What''s UP' AND `tag` != 'test'", }, { name: "doris t8est text field wildcard", From 5ac4b188c42125cd6fa77f0cab470a6dd15ff645 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 20 Mar 2025 22:26:17 +0800 Subject: [PATCH 036/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format_test.go | 201 +++++++++++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 2 +- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 10 +- 3 files changed, 205 insertions(+), 8 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 717cb9e19..5b6a7f2d6 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -11,6 +11,7 @@ package bksql_test import ( "context" + "fmt" "testing" "time" @@ -18,6 +19,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" ) @@ -64,7 +66,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 0, Orders: metadata.Orders{"level": true}, }, - expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-count_over_time-with-promql-seconds": { query: &metadata.Query{ @@ -143,7 +145,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -175,7 +177,44 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + }, + "doris count by day with UTC": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour * 24, + }, + }, + }, + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1440 AS INT) * 1440 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + "doris count by day with Asia/Shanghai": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour * 24, + TimeZone: "Asia/ShangHai", + }, + }, + }, + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1440 AS INT) * 1440 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { @@ -195,3 +234,159 @@ func TestNewSqlFactory(t *testing.T) { }) } } + +func TestWindowWithTimezone(t *testing.T) { + mock.Init() + + testCases := []struct { + name string + start time.Time + timezone string + window time.Duration + + expected time.Time + }{ + { + name: "test window 1m - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/ShangHai", + window: time.Minute, + + expected: time.UnixMilli(1742267700000), + }, + { + name: "test window 1d +8 - 1", + start: time.UnixMilli(1742267704000), + window: time.Hour * 24, + + expected: time.UnixMilli(1742256000000), + }, + { + name: "test window 1d +6 - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/Urumqi", + window: time.Hour * 24, + + expected: time.UnixMilli(1742234400000), + }, + { + name: "test window 1d -6 - 1", + start: time.UnixMilli(1742267704000), + timezone: "America/Guatemala", + window: time.Hour * 24, + + expected: time.UnixMilli(1742191200000), + }, + { + name: "test window 1d +8 - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/ShangHai", + window: time.Hour * 24, + + expected: time.UnixMilli(1742227200000), + }, + { + name: "test window 1d +8 - 2", + start: time.UnixMilli(1741885200000), + timezone: "Asia/ShangHai", + window: time.Hour * 24, + + expected: time.UnixMilli(1741881600000), + }, + { + name: "test window 26h +8 - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/ShangHai", + window: time.Hour*24 + time.Hour*2, + + expected: time.UnixMilli(1742176800000), + }, + { + name: "test window 3d +8 - 1", + start: time.UnixMilli(1742267704000), + timezone: "Asia/ShangHai", + window: time.Hour * 24 * 3, + + expected: time.UnixMilli(1742054400000), + }, + { + name: "test window 1m +8 - 2", + start: time.UnixMilli(1742266099000), + timezone: "Asia/ShangHai", + window: time.Minute, + + expected: time.UnixMilli(1742266080000), + }, + { + name: "test window 6h +8 - 2", + start: time.UnixMilli(1742266099000), // 2025-03-18 10:48:19 +0800 + timezone: "Asia/ShangHai", + window: time.Hour * 6, + + expected: time.UnixMilli(1742256000000), // 2025-03-18 08:00:00 +0800 CST + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cTime := tc.start.UnixMilli() + window := tc.window.Milliseconds() + + var offset int64 + loc, _ := time.LoadLocation(tc.timezone) + if window%(time.Hour*24).Milliseconds() == 0 { + _, z := time.Now().In(loc).Zone() + offset = int64(z) * 1000 + } + + nTime1 := (cTime + offset) - (cTime+offset)%window - offset + nTime2 := (cTime+offset)/window*window - offset + + var milliUnixToString = func(milliUnix int64) string { + return time.UnixMilli(milliUnix).In(loc).String() + } + + log.Infof(context.TODO(), "window: %d, nTime: %s, nTime1: %s nTime2: %s", window/1000, milliUnixToString(cTime), milliUnixToString(nTime1), milliUnixToString(nTime2)) + log.Infof(context.TODO(), "nTime1: (%d + %d) - (%d + %d) %% %d - %d", cTime, offset, cTime, offset, window, offset) + log.Infof(context.TODO(), "nTime2: (%d + %d) / %d * %d - %d", cTime, offset, window, window, offset) + + assert.Equal(t, nTime1, nTime2) + assert.Equal(t, tc.expected.UnixMilli(), nTime1) + }) + } +} + +// TestTimeZone 时区聚合测试 +func TestTimeZone(t *testing.T) { + timezones := []string{ + "Asia/Shanghai", + //"America/New_York", + //"Pacific/Auckland", + //"Europe/London", + //"UTC", + } + + for _, tz := range timezones { + loc, _ := time.LoadLocation(tz) + _, z := time.Now().In(loc).Zone() + offset := int64(z) * 1000 + window := (time.Hour * 24).Milliseconds() + + fmt.Println("------------------------------------") + fmt.Println(tz, offset/1000/60/60) + fmt.Println("------------------------------------") + + st := time.Date(2025, 3, 18, 0, 0, 0, 0, time.UTC) + for i := 0; i < 50; i++ { + //milli := (st.UnixMilli() + offset) - (st.UnixMilli()+offset)%window - offset + milli := (st.UnixMilli())/window*window - offset + ct := st.In(loc).String() + ot := time.UnixMilli(milli).In(loc).String() + fmt.Println(st, "=>", ct, "=>", ot) + + st = st.Add(time.Hour) + } + fmt.Println("------------------------------------") + } + +} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 5af5d75c2..9e6d44f2b 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -167,7 +167,7 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate offsetMillis = offset * 1e3 } - timeField := fmt.Sprintf("(`%s` - ((`%s` - %d) %% %d - %d))", d.timeField, d.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) + timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, d.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) groupByFields = append(groupByFields, timeField) selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 4ece72a6e..215c49275 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -95,21 +95,23 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, if agg.Window > 0 { // 获取时区偏移量 - var offset int + var offsetMinutes int + // 如果是按天聚合，则增加时区偏移量 if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { // 时间聚合函数兼容时区 loc, locErr := time.LoadLocation(agg.TimeZone) if locErr != nil { loc = time.UTC } - _, offset = time.Now().In(loc).Zone() - fmt.Println(offset) + _, offset := time.Now().In(loc).Zone() + offsetMinutes = offset / 60 } // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 var timeField string if int64(agg.Window.Seconds())%60 == 0 { - timeField = fmt.Sprintf(`CAST(%s / 1000 / %.f AS INT) * %.f * 60 * 1000`, ShardKey, agg.Window.Minutes(), agg.Window.Minutes()) + windowMinutes := int(agg.Window.Minutes()) + timeField = fmt.Sprintf(`((CAST((%s / 1000 + %d) / %d AS INT) * %d - %d) * 60 * 1000)`, ShardKey, offsetMinutes, windowMinutes, windowMinutes, offsetMinutes) } else { timeField = fmt.Sprintf(`CAST(%s / %d AS INT) * %d `, d.timeField, agg.Window.Milliseconds(), agg.Window.Milliseconds()) } From f16dea38e609c78d75e79a14c8048d0fadc1dfa1 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 21 Mar 2025 19:36:58 +0800 Subject: [PATCH 037/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query_test.go | 8 ++ pkg/unify-query/tsdb/bksql/format.go | 29 ++++--- pkg/unify-query/tsdb/bksql/format_test.go | 10 +-- pkg/unify-query/tsdb/bksql/instance.go | 85 +++++++++++++++++++-- pkg/unify-query/tsdb/bksql/instance_test.go | 32 ++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 8 +- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 4 + 7 files changed, 144 insertions(+), 32 deletions(-) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 2ecad452f..5247aed1b 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -14,6 +14,7 @@ import ( "fmt" "sort" "strconv" + "strings" "testing" "time" @@ -2497,6 +2498,13 @@ func TestQueryTsClusterMetrics(t *testing.T) { assert.Nil(t, err) res, err := QueryTsClusterMetrics(ctx, query) + + pd := res.(*PromData) + + sort.SliceStable(pd.Tables, func(i, j int) bool { + return strings.Join(pd.Tables[i].GroupKeys, "") < strings.Join(pd.Tables[j].GroupKeys, "") + }) + assert.Nil(t, err) out, err := json.Marshal(res) assert.Nil(t, err) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index eda194b6c..b0911747d 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -90,17 +90,29 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { return f } -func (f *QueryFactory) WithFieldsMap(fieldsMap map[string]string) *QueryFactory { - f.expr.WithFieldsMap(fieldsMap) - return f -} - func (f *QueryFactory) WithRangeTime(start, end time.Time) *QueryFactory { f.start = start f.end = end return f } +func (f *QueryFactory) WithFieldsMap(m map[string]string) *QueryFactory { + f.expr.WithFieldsMap(m) + return f +} + +func (f *QueryFactory) Table() string { + table := fmt.Sprintf("`%s`", f.query.DB) + if f.query.Measurement != "" { + table += "." + f.query.Measurement + } + return table +} + +func (f *QueryFactory) DescribeTable() string { + return f.expr.DescribeTable(f.Table()) +} + func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") @@ -171,12 +183,7 @@ func (f *QueryFactory) SQL() (sql string, err error) { return } - table := fmt.Sprintf("`%s`", f.query.DB) - if f.query.Measurement != "" { - table += "." + f.query.Measurement - } - - sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), table) + sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), f.Table()) whereString, err := f.BuildWhere() if err != nil { return diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 5b6a7f2d6..14559e42f 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -66,7 +66,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 0, Orders: metadata.Orders{"level": true}, }, - expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-count_over_time-with-promql-seconds": { query: &metadata.Query{ @@ -145,7 +145,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -177,7 +177,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1 AS INT) * 1 * 1000 + 0) * 60 AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count by day with UTC": { // 2025-03-14 15:05:45 Asia/ShangHai @@ -195,7 +195,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1440 AS INT) * 1440 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count by day with Asia/Shanghai": { // 2025-03-14 15:05:45 Asia/ShangHai @@ -214,7 +214,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, (CAST((__shard_key__ / 1000 - 0) / 1440 AS INT) * 1440 * 1000 + 0) * 60 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 480) / 1440 AS INT) * 1440 - 480) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 8fa91aa64..401352025 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -38,6 +38,9 @@ const ( KeyIndex = "__index" KeyTableID = "__result_table" KeyDataLabel = "__data_label" + + TableFieldName = "Field" + TableFieldType = "Type" ) type Instance struct { @@ -116,6 +119,10 @@ func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) ( err error ) + if sql == "" { + return data, nil + } + log.Infof(ctx, "%s: %s", i.InstanceType(), sql) span.Set("query-sql", sql) @@ -271,9 +278,62 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad return res, nil } -func (i *Instance) table(query *metadata.Query) string { +func (i *Instance) getFieldsMap(ctx context.Context, sql string, span *trace.Span) (map[string]string, error) { + fieldsMap := make(map[string]string) + + if sql == "" { + return nil, nil + } + + data, err := i.sqlQuery(ctx, sql, span) + if err != nil { + return nil, err + } + + for _, list := range data.List { + var ( + k string + v string + ok bool + ) + k, ok = list[TableFieldName].(string) + if !ok { + continue + } + + v, ok = list[TableFieldType].(string) + if !ok { + continue + } + fieldsMap[k] = v + } + + return fieldsMap, nil +} + +func (i *Instance) InitQueryFactory(ctx context.Context, query *metadata.Query, start, end time.Time, span *trace.Span) (*QueryFactory, error) { + + f := NewQueryFactory(ctx, query).WithRangeTime(start, end) + + // 只有 Doris 才需要获取字段表结构 + if query.Measurement == sqlExpr.Doris { + fieldsMap, err := i.getFieldsMap(ctx, f.DescribeTable(), span) + if err != nil { + return f, err + } + + span.Set("table_fields_map", fieldsMap) + + f.WithFieldsMap(fieldsMap) + } + + return f, nil +} + +func (i *Instance) Table(query *metadata.Query) string { table := fmt.Sprintf("`%s`", query.DB) if query.Measurement != "" { + table += "." + query.Measurement } return table } @@ -307,8 +367,10 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } } - queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) - + queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + if err != nil { + return + } sql, err := queryFactory.SQL() if err != nil { return @@ -371,8 +433,10 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } } - queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) - + queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + if err != nil { + return storage.ErrSeriesSet(err) + } sql, err := queryFactory.SQL() if err != nil { return storage.ErrSeriesSet(err) @@ -428,7 +492,11 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s // 取字段名不需要返回数据，但是 size 不能使用 0，所以还是用 1 query.Size = 1 - queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + if err != nil { + return nil, err + } + sql, err := queryFactory.SQL() if err != nil { return nil, err @@ -465,7 +533,10 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, }, } - queryFactory := NewQueryFactory(ctx, query).WithRangeTime(start, end) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + if err != nil { + return nil, err + } sql, err := queryFactory.SQL() if err != nil { return nil, err diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 79939a597..19973aba4 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -32,9 +32,11 @@ func TestInstance_QueryRaw(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", - "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", - "SELECT COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) end := time.UnixMilli(1730118889181) @@ -102,6 +104,18 @@ func TestInstance_QueryRaw(t *testing.T) { }, expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118600000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118660000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118720000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118780000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118840000,"_value_":11}]`, }, + "count by doris": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + Size: 2, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext":"{\"container_id\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"container_image\":\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\",\"container_name\":\"bkmonitorbeat\",\"io_kubernetes_pod\":\"bkm-daemonset-worker-9tckj\",\"io_kubernetes_pod_namespace\":\"bkmonitor-operator\",\"io_kubernetes_pod_uid\":\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\",\"io_kubernetes_workload_name\":\"bkm-daemonset-worker\",\"io_kubernetes_workload_type\":\"DaemonSet\",\"labels\":{\"app_kubernetes_io_component\":\"bkmonitorbeat\",\"controller_revision_hash\":\"6b87cb95fc\",\"pod_template_generation\":\"14\"}}","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext":"{\"container_id\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"container_image\":\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\",\"container_name\":\"bkmonitorbeat\",\"io_kubernetes_pod\":\"bkm-daemonset-worker-9tckj\",\"io_kubernetes_pod_namespace\":\"bkmonitor-operator\",\"io_kubernetes_pod_uid\":\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\",\"io_kubernetes_workload_name\":\"bkm-daemonset-worker\",\"io_kubernetes_workload_type\":\"DaemonSet\",\"labels\":{\"app_kubernetes_io_component\":\"bkmonitorbeat\",\"controller_revision_hash\":\"6b87cb95fc\",\"pod_template_generation\":\"14\"}}","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, } { t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) @@ -131,7 +145,7 @@ func TestInstance_QueryRaw(t *testing.T) { actual, err := json.Marshal(list) assert.Nil(t, err) - assert.Equal(t, c.expected, string(actual)) + assert.JSONEq(t, c.expected, string(actual)) }) } } @@ -171,7 +185,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0))) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 15000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 15000 * 15000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (dtEventTimeStamp + 0) / 15000 * 15000 - 0 ORDER BY `_timestamp_` ASC", }, { query: &metadata.Query{ @@ -378,7 +392,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0))) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 3600000 - 0)) ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 3600000 * 3600000 - 0) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (dtEventTimeStamp + 0) / 3600000 * 3600000 - 0 ORDER BY `_timestamp_` ASC", }, } @@ -636,7 +650,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(__shard_key__ / 1000 / 1 AS INT) * 1 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例11: doris 处理 object 字段 + 时间聚合 5m { @@ -660,7 +674,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(__shard_key__ / 1000 / 5 AS INT) * 5 * 60 * 1000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例12: doris 处理 object 字段 + 时间聚合 15s { @@ -737,6 +751,7 @@ func TestInstance_QueryLabelNames_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ + "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, }) @@ -788,6 +803,7 @@ func TestInstance_QueryLabelValues_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ + "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, }) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 9e6d44f2b..34ff9998d 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -49,6 +49,8 @@ type SQLExpr interface { ParserAllConditions(allConditions metadata.AllConditions) (string, error) // ParserAggregatesAndOrders 解析聚合条件生成SQL条件表达式 ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, error) + // DescribeTable 返回当前表结构 + DescribeTable(table string) string } // SQL表达式注册管理相关变量 @@ -167,7 +169,7 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate offsetMillis = offset * 1e3 } - timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, d.timeField, offsetMillis, agg.Window.Milliseconds(), offsetMillis) + timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, offsetMillis, agg.Window.Milliseconds(), agg.Window.Milliseconds(), offsetMillis) groupByFields = append(groupByFields, timeField) selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) @@ -251,6 +253,10 @@ func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllCondition return "", nil } +func (d *DefaultSQLExpr) DescribeTable(table string) string { + return "" +} + // buildCondition 构建单个条件表达式 func (d *DefaultSQLExpr) buildCondition(c metadata.ConditionField) (string, error) { if len(c.Value) == 0 { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 215c49275..df94bdd8b 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -64,6 +64,10 @@ func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { return d.walk(expr) } +func (d *DorisSQLExpr) DescribeTable(table string) string { + return fmt.Sprintf("SHOW CREATE %s", table) +} + // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { valueField, _ := d.dimTransform(d.valueField) From c248e02da315d7f408e80b84f2bb039557215f09 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 26 Mar 2025 16:53:09 +0800 Subject: [PATCH 038/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 96b188430..09fd92e8b 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -12,6 +12,7 @@ package http import ( "context" "fmt" + "sort" "strconv" "strings" "testing" From b8fc58291e477ea986a2e44e929675da3a1b8874 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 27 Mar 2025 11:05:19 +0800 Subject: [PATCH 039/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/condition.go | 20 +++++++ pkg/unify-query/query/structured/query_ts.go | 55 ++++++------------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/pkg/unify-query/query/structured/condition.go b/pkg/unify-query/query/structured/condition.go index e06452aba..6daf4d6f9 100644 --- a/pkg/unify-query/query/structured/condition.go +++ b/pkg/unify-query/query/structured/condition.go @@ -198,6 +198,26 @@ func MergeConditionField(source, target AllConditions) AllConditions { return all } +func (c AllConditions) MetaDataAllConditions() metadata.AllConditions { + var allConditions metadata.AllConditions + if len(c) > 0 { + allConditions = make(metadata.AllConditions, 0, len(c)) + for _, conditions := range c { + conds := make([]metadata.ConditionField, 0, len(conditions)) + for _, c := range conditions { + conds = append(conds, metadata.ConditionField{ + DimensionName: c.DimensionName, + Value: c.Value, + Operator: c.Operator, + IsWildcard: c.IsWildcard, + }) + } + allConditions = append(allConditions, conds) + } + } + return allConditions +} + func (c AllConditions) BkSql() string { var conditionsString []string for _, cond := range c { diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 6ae0adcab..26af1b0b2 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -480,6 +480,11 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q return nil, err } + allConditions, err := q.Conditions.AnalysisConditions() + if err != nil { + return nil, err + } + // 如果 DataSource 为空，则自动补充 if q.DataSource == "" { q.DataSource = BkMonitor @@ -487,12 +492,6 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q // 如果是 BkSql 查询无需获取 tsdb 路由关系 if q.DataSource == BkData { - allConditions, bkDataErr := q.Conditions.AnalysisConditions() - if bkDataErr != nil { - err = bkDataErr - return nil, bkDataErr - } - // 判断空间跟业务是否匹配 isMatchBizID := func() bool { space := strings.Split(spaceUid, "__") @@ -528,17 +527,17 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q } qry := &metadata.Query{ - StorageType: consul.BkSqlStorageType, - TableID: string(tableID), - DataSource: q.DataSource, - DB: route.DB(), - Measurement: route.Measurement(), - Field: q.FieldName, - MetricName: metricName, - Aggregates: aggregates, - BkSqlCondition: allConditions.BkSql(), - Size: q.Limit, - From: q.From, + StorageType: consul.BkSqlStorageType, + TableID: string(tableID), + DataSource: q.DataSource, + DB: route.DB(), + Measurement: route.Measurement(), + Field: q.FieldName, + MetricName: metricName, + Aggregates: aggregates, + AllConditions: allConditions.MetaDataAllConditions(), + Size: q.Limit, + From: q.From, } if len(q.OrderBy) > 0 { @@ -580,11 +579,6 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q isSkipField = true } - allConditions, err := q.Conditions.AnalysisConditions() - if err != nil { - return nil, err - } - tsDBs, err := GetTsDBList(ctx, &TsDBOption{ SpaceUid: spaceUid, TableID: tableID, @@ -907,6 +901,7 @@ func (q *Query) BuildMetadataQuery( query.NeedAddTime = tsDB.NeedAddTime query.SourceType = tsDB.SourceType + query.AllConditions = allCondition.MetaDataAllConditions() query.Condition = whereList.String() query.VmCondition, query.VmConditionNum = allCondition.VMString(query.VmRt, vmMetric, q.IsRegexp) @@ -917,22 +912,6 @@ func (q *Query) BuildMetadataQuery( query.Size = q.Limit query.From = q.From - if len(allCondition) > 0 { - query.AllConditions = make(metadata.AllConditions, len(allCondition)) - for i, conditions := range allCondition { - conds := make([]metadata.ConditionField, len(conditions)) - for j, c := range conditions { - conds[j] = metadata.ConditionField{ - DimensionName: c.DimensionName, - Value: c.Value, - Operator: c.Operator, - IsWildcard: c.IsWildcard, - } - } - query.AllConditions[i] = conds - } - } - if len(q.OrderBy) > 0 { query.Orders = q.OrderBy.Orders() } From 6573b81ae977f8b39add0ec78bdcc73575448650 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 27 Mar 2025 22:45:03 +0800 Subject: [PATCH 040/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/bkapi/bkapi_test.go | 8 +- .../metadata/result_table_options.go | 19 ++++ pkg/unify-query/metadata/struct.go | 12 ++- pkg/unify-query/query/structured/query_ts.go | 30 ++++++- pkg/unify-query/service/http/handler.go | 2 +- pkg/unify-query/service/http/query.go | 25 ++++-- pkg/unify-query/service/http/query_test.go | 7 +- pkg/unify-query/service/http/response.go | 9 +- pkg/unify-query/tsdb/bksql/instance.go | 4 +- .../tsdb/elasticsearch/instance.go | 89 +++++++++++++------ .../tsdb/elasticsearch/instance_test.go | 2 +- pkg/unify-query/tsdb/influxdb/instance.go | 4 +- pkg/unify-query/tsdb/interfaces.go | 2 +- .../tsdb/offlineDataArchive/instance.go | 4 +- pkg/unify-query/tsdb/prometheus/instance.go | 4 +- pkg/unify-query/tsdb/redis/instance.go | 29 +++--- .../tsdb/victoriaMetrics/instance.go | 4 +- 17 files changed, 169 insertions(+), 85 deletions(-) create mode 100644 pkg/unify-query/metadata/result_table_options.go diff --git a/pkg/unify-query/bkapi/bkapi_test.go b/pkg/unify-query/bkapi/bkapi_test.go index 09dccceff..d5db754cf 100644 --- a/pkg/unify-query/bkapi/bkapi_test.go +++ b/pkg/unify-query/bkapi/bkapi_test.go @@ -10,6 +10,7 @@ package bkapi import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -30,8 +31,7 @@ func TestGetBkAPI(t *testing.T) { "Content-Type": "application/json", }) - assert.Equal(t, map[string]string{ - "Content-Type": "application/json", - "X-Bkapi-Authorization": `{"bk_username":"admin","bk_app_code":"bk_code","bk_app_secret":"bk_secret"}`, - }, headers) + actual, _ := json.Marshal(headers) + + assert.JSONEq(t, `{"Content-Type":"application/json","X-Bkapi-Authorization":"{\"bk_username\":\"admin\",\"bk_app_code\":\"bk_code\",\"bk_app_secret\":\"bk_secret\"}"}`, string(actual)) } diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go new file mode 100644 index 000000000..309a94be5 --- /dev/null +++ b/pkg/unify-query/metadata/result_table_options.go @@ -0,0 +1,19 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package metadata + +type ResultTableOptions map[string]ResultTableOption + +type ResultTableOption struct { + Total int64 `json:"total,omitempty"` + From int `json:"from,omitempty"` + ScrollID string `json:"scroll_id,omitempty"` + SearchAfter []any `json:"search_after,omitempty"` +} diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 1495655e9..1ae5fc2b2 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -117,11 +117,15 @@ type Query struct { QueryString string AllConditions AllConditions - HighLight HighLight + HighLight *HighLight + + Source []string + From int + Size int + + Scroll string + ResultTableOptions ResultTableOptions - Source []string - From int - Size int Orders Orders NeedAddTime bool } diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 38b03c297..d293b5bf4 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -65,8 +65,14 @@ type QueryTs struct { Limit int `json:"limit,omitempty" example:"0"` // From 翻页开启数字 From int `json:"from,omitempty" example:"0"` + + // Scroll 是否启用 Scroll 查询 + Scroll string `json:"scroll,omitempty"` + + ResultTableOptions metadata.ResultTableOptions `json:"result_table_options,omitempty"` + // HighLight 是否开启高亮 - HighLight metadata.HighLight `json:"highlight,omitempty"` + HighLight *metadata.HighLight `json:"highlight,omitempty"` } // StepParse 解析step @@ -133,9 +139,19 @@ func (q *QueryTs) ToQueryReference(ctx context.Context) (metadata.QueryReference q.SpaceUid = metadata.GetUser(ctx).SpaceUid } + if q.ResultTableOptions != nil { + query.ResultTableOptions = q.ResultTableOptions + } + + if q.Scroll != "" { + query.Scroll = q.Scroll + } + // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if !query.HighLight.Enable && q.HighLight.Enable { - query.HighLight = q.HighLight + if query.HighLight != nil { + if !query.HighLight.Enable && q.HighLight.Enable { + query.HighLight = q.HighLight + } } // 复用字段配置，没有特殊配置的情况下使用公共配置 @@ -379,8 +395,12 @@ type Query struct { // IsReference 是否使用非时间聚合查询 IsReference bool `json:"-" swaggerignore:"true"` + Scroll string `json:"scroll,omitempty"` + // ResultTableOptions + ResultTableOptions metadata.ResultTableOptions `json:"result_table_options,omitempty"` + // HighLight 是否打开高亮，只对原始数据接口生效 - HighLight metadata.HighLight `json:"highlight,omitempty"` + HighLight *metadata.HighLight `json:"highlight,omitempty"` } func (q *Query) ToRouter() (*Route, error) { @@ -893,6 +913,8 @@ func (q *Query) BuildMetadataQuery( query.HighLight = q.HighLight query.Size = q.Limit query.From = q.From + query.Scroll = q.Scroll + query.ResultTableOptions = q.ResultTableOptions if len(allCondition) > 0 { query.AllConditions = make(metadata.AllConditions, len(allCondition)) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index edfb01327..f4402e91f 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -238,7 +238,7 @@ func HandlerQueryRaw(c *gin.Context) { span.Set("query-body", string(queryStr)) listData.TraceID = span.TraceID() - listData.Total, listData.List, err = queryRawWithInstance(ctx, queryTs) + listData.Total, listData.List, listData.ResultTableOptions, err = queryRawWithInstance(ctx, queryTs) if err != nil { listData.Status = &metadata.Status{ Code: metadata.QueryRawError, diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 637acbe9f..a82accc30 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -157,7 +157,7 @@ func queryExemplar(ctx context.Context, query *structured.QueryTs) (interface{}, return resp, err } -func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (total int64, list []map[string]any, err error) { +func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (total int64, list []map[string]any, resultTableOptions metadata.ResultTableOptions, err error) { ctx, span := trace.NewSpan(ctx, "query-raw-with-instance") defer span.End(&err) @@ -180,6 +180,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot receiveWg.Add(1) list = make([]map[string]any, 0) + resultTableOptions = make(metadata.ResultTableOptions) + // 启动合并数据 go func() { defer receiveWg.Done() @@ -232,9 +234,13 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot ql.From = queryTs.From } + // 复用 Scroll 配置 + // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if !ql.HighLight.Enable && queryTs.HighLight.Enable { - ql.HighLight = queryTs.HighLight + if ql.HighLight != nil { + if !ql.HighLight.Enable && queryTs.HighLight.Enable { + ql.HighLight = queryTs.HighLight + } } // 复用字段配置，没有特殊配置的情况下使用公共配置 @@ -265,13 +271,18 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot return } - size, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) + options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) + lock.Lock() if queryErr != nil { - lock.Lock() message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) - lock.Unlock() + } else { + // 合并 ResultTableOptions + for k, v := range options { + resultTableOptions[k] = v + total += v.Total + } } - atomic.AddInt64(&total, size) + lock.Unlock() }) } } diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 60204565c..3847f1066 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -724,6 +724,7 @@ func TestQueryRawWithInstance(t *testing.T) { queryTs *structured.QueryTs total int64 expected string + options string }{ "query with EpochMillis": { queryTs: &structured.QueryTs{ @@ -742,6 +743,7 @@ func TestQueryRawWithInstance(t *testing.T) { }, total: 1e4, expected: `[{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__doc_id":"c124dae69af9b86a7128ee4281820158","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"0826407be7f04f19086774ed68eac8dd"},{"__data_label":"bkbase_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"}]`, + options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":11}}`, }, "query_bk_base_es_with_raw": { queryTs: &structured.QueryTs{ @@ -794,7 +796,7 @@ func TestQueryRawWithInstance(t *testing.T) { for name, c := range tcs { t.Run(name, func(t *testing.T) { - total, list, err := queryRawWithInstance(ctx, c.queryTs) + total, list, options, err := queryRawWithInstance(ctx, c.queryTs) assert.Nil(t, err) if err != nil { return @@ -803,6 +805,9 @@ func TestQueryRawWithInstance(t *testing.T) { assert.Equal(t, c.total, total) actual, _ := json.Marshal(list) assert.JSONEq(t, c.expected, string(actual)) + + optActual, _ := json.Marshal(options) + assert.JSONEq(t, c.options, string(optActual)) }) } } diff --git a/pkg/unify-query/service/http/response.go b/pkg/unify-query/service/http/response.go index cc50d8a12..fef3dc848 100644 --- a/pkg/unify-query/service/http/response.go +++ b/pkg/unify-query/service/http/response.go @@ -48,8 +48,9 @@ func (r *response) success(ctx context.Context, data interface{}) { // ListData 数据返回格式 type ListData struct { - Total int64 `json:"total,omitempty"` - List []map[string]any `json:"list" json:"list,omitempty"` - TraceID string `json:"trace_id,omitempty"` - Status *metadata.Status `json:"status,omitempty" json:"status,omitempty"` + Total int64 `json:"total,omitempty"` + List []map[string]any `json:"list" json:"list,omitempty"` + TraceID string `json:"trace_id,omitempty"` + Status *metadata.Status `json:"status,omitempty" json:"status,omitempty"` + ResultTableOptions metadata.ResultTableOptions `json:"result_table_options,omitempty"` } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 4e28715b8..7d1befbcd 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -276,8 +276,8 @@ func (i *Instance) table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 5fde6b3f0..0bbc2e928 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -17,7 +17,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" elastic "github.com/olivere/elastic/v7" @@ -270,7 +269,7 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac fact.Size(source) } - if qb.HighLight.Enable { + if qb.HighLight != nil && qb.HighLight.Enable { source.Highlight(fact.HighLight(qb.QueryString, qb.HighLight.MaxAnalyzedOffset)) } @@ -301,9 +300,26 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac if err != nil { return nil, err } - search := client.Search().Index(qo.indexes...).SearchSource(source) - res, err := search.Do(ctx) + var res *elastic.SearchResult + func() { + if opt, ok := qb.ResultTableOptions[qb.TableID+"|"+qo.conn.Address]; ok { + if opt.ScrollID != "" { + res, err = client.Scroll(qo.indexes...).ScrollId(opt.ScrollID).Do(ctx) + return + } + + if len(opt.SearchAfter) > 0 { + return + } + } + + if qb.Scroll != "" { + res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).SearchSource(source).Do(ctx) + } else { + res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) + } + }() if err != nil { var ( @@ -526,11 +542,12 @@ func (i *Instance) getAlias(ctx context.Context, db string, needAddTime bool, st } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { var ( - err error - total int64 - wg sync.WaitGroup + err error + wg sync.WaitGroup + + resultTableOptions = make(metadata.ResultTableOptions) ) defer func() { @@ -546,13 +563,13 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star if query.DB == "" { err = fmt.Errorf("%s 查询别名为空", query.TableID) - return total, err + return resultTableOptions, err } unit := metadata.GetQueryParams(ctx).TimeUnit aliases, err := i.getAlias(ctx, query.DB, query.NeedAddTime, start, end, query.Timezone) if err != nil { - return total, err + return resultTableOptions, err } for _, conn := range i.connects { @@ -593,36 +610,50 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star err = queryErr return } - for _, d := range sr.Hits.Hits { - data := make(map[string]any) - if err = json.Unmarshal(d.Source, &data); err != nil { - return + + option := metadata.ResultTableOption{} + if sr != nil { + if sr.ScrollId != "" { + option.ScrollID = sr.ScrollId } - fact.SetData(data) - fact.data[KeyDocID] = d.Id - fact.data[KeyIndex] = d.Index - fact.data[KeyTableID] = query.TableID - fact.data[KeyDataLabel] = query.DataLabel + if sr.Hits != nil { + for _, d := range sr.Hits.Hits { + data := make(map[string]any) + if err = json.Unmarshal(d.Source, &data); err != nil { + return + } + + fact.SetData(data) + fact.data[KeyDocID] = d.Id + fact.data[KeyIndex] = d.Index + fact.data[KeyTableID] = query.TableID + fact.data[KeyDataLabel] = query.DataLabel + + if timeValue, ok := data[fact.GetTimeField().Name]; ok { + fact.data[FieldTime] = timeValue + } + + if len(d.Highlight) > 0 { + fact.data[KeyHighLight] = d.Highlight + } + dataCh <- fact.data + } - if timeValue, ok := data[fact.GetTimeField().Name]; ok { - fact.data[FieldTime] = timeValue - } + option.From = query.From + len(sr.Hits.Hits) - if len(d.Highlight) > 0 { - fact.data[KeyHighLight] = d.Highlight + if sr.Hits.TotalHits != nil { + option.Total = sr.Hits.TotalHits.Value + } } - dataCh <- fact.data } - if sr != nil && sr.Hits != nil && sr.Hits.TotalHits != nil { - atomic.AddInt64(&total, sr.Hits.TotalHits.Value) - } + resultTableOptions[query.TableID+"|"+conn.Address] = option }() } wg.Wait() - return total, err + return resultTableOptions, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index 324a32cc9..fb1740a02 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -129,7 +129,7 @@ func TestInstance_queryReference(t *testing.T) { }, }, QueryString: "group: fans", - HighLight: metadata.HighLight{ + HighLight: &metadata.HighLight{ Enable: true, }, }, diff --git a/pkg/unify-query/tsdb/influxdb/instance.go b/pkg/unify-query/tsdb/influxdb/instance.go index e19d63505..1dadbd3cd 100644 --- a/pkg/unify-query/tsdb/influxdb/instance.go +++ b/pkg/unify-query/tsdb/influxdb/instance.go @@ -649,8 +649,8 @@ func (i *Instance) grpcStream( } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/interfaces.go b/pkg/unify-query/tsdb/interfaces.go index 379702f54..e8830a590 100644 --- a/pkg/unify-query/tsdb/interfaces.go +++ b/pkg/unify-query/tsdb/interfaces.go @@ -22,7 +22,7 @@ import ( ) type Instance interface { - QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) + QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) diff --git a/pkg/unify-query/tsdb/offlineDataArchive/instance.go b/pkg/unify-query/tsdb/offlineDataArchive/instance.go index 41b313fc7..eeff6eb63 100644 --- a/pkg/unify-query/tsdb/offlineDataArchive/instance.go +++ b/pkg/unify-query/tsdb/offlineDataArchive/instance.go @@ -112,8 +112,8 @@ func (i Instance) setClient() error { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/prometheus/instance.go b/pkg/unify-query/tsdb/prometheus/instance.go index 5e0ed418a..0676e245e 100644 --- a/pkg/unify-query/tsdb/prometheus/instance.go +++ b/pkg/unify-query/tsdb/prometheus/instance.go @@ -69,8 +69,8 @@ func (i *Instance) InstanceType() string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/redis/instance.go b/pkg/unify-query/tsdb/redis/instance.go index 882809faa..7ffab88cf 100644 --- a/pkg/unify-query/tsdb/redis/instance.go +++ b/pkg/unify-query/tsdb/redis/instance.go @@ -43,49 +43,40 @@ type Instance struct { ClusterMetricPrefix string } -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - //TODO implement me - panic("implement me") +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } func (i *Instance) Check(ctx context.Context, promql string, start, end time.Time, step time.Duration) string { - //TODO implement me - panic("implement me") + return "" } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { - //TODO implement me - panic("implement me") + return nil } func (i *Instance) QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, start, end time.Time) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, name string, start, end time.Time) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) QuerySeries(ctx context.Context, query *metadata.Query, start, end time.Time) ([]map[string]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelNames(ctx context.Context, start, end time.Time, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) DirectLabelValues(ctx context.Context, name string, start, end time.Time, limit int, matchers ...*labels.Matcher) ([]string, error) { - //TODO implement me - panic("implement me") + return nil, nil } func (i *Instance) InstanceType() string { diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance.go b/pkg/unify-query/tsdb/victoriaMetrics/instance.go index f92e9918f..5bad2022b 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance.go @@ -115,8 +115,8 @@ func (i *Instance) Check(ctx context.Context, q string, start, end time.Time, st } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, error) { - return 0, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { + return nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 From 45859593471feb8a0432e86dd782653f1ad68dde Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 31 Mar 2025 17:43:52 +0800 Subject: [PATCH 041/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata/result_table_options.go | 44 +++- pkg/unify-query/query/structured/query_ts.go | 11 +- .../query/structured/query_ts_test.go | 7 +- pkg/unify-query/service/http/query.go | 192 ++++++++++-------- pkg/unify-query/service/http/query_test.go | 43 ++-- pkg/unify-query/tsdb/elasticsearch/format.go | 6 +- .../tsdb/elasticsearch/instance.go | 42 +++- 7 files changed, 225 insertions(+), 120 deletions(-) diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index 309a94be5..854e3b024 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -9,7 +9,7 @@ package metadata -type ResultTableOptions map[string]ResultTableOption +type ResultTableOptions map[string]*ResultTableOption type ResultTableOption struct { Total int64 `json:"total,omitempty"` @@ -17,3 +17,45 @@ type ResultTableOption struct { ScrollID string `json:"scroll_id,omitempty"` SearchAfter []any `json:"search_after,omitempty"` } + +func (o ResultTableOptions) MergeOptions(options ResultTableOptions) { + for k, v := range options { + if s, ok := o[k]; ok { + v.Total += s.Total + v.From += s.From + + // ScrollID 和 SearchAfter 保持不变 + v.ScrollID = s.ScrollID + v.SearchAfter = s.SearchAfter + } + o[k] = v + } +} + +func (o ResultTableOptions) SetOption(tableID, address string, option *ResultTableOption) { + o[tableID+"|"+address] = option +} + +func (o ResultTableOptions) GetOption(tableID, address string) *ResultTableOption { + if option, ok := o[tableID+"|"+address]; ok { + return option + } + return &ResultTableOption{} +} + +func (o ResultTableOptions) GetTotal() int64 { + var total int64 + for _, v := range o { + total += v.Total + } + return total +} + +func (o ResultTableOptions) IsMultiFrom() bool { + for _, v := range o { + if v.ScrollID != "" || len(v.SearchAfter) > 0 { + return false + } + } + return true +} diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index d293b5bf4..6d0bcf84b 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -395,9 +395,10 @@ type Query struct { // IsReference 是否使用非时间聚合查询 IsReference bool `json:"-" swaggerignore:"true"` - Scroll string `json:"scroll,omitempty"` // ResultTableOptions - ResultTableOptions metadata.ResultTableOptions `json:"result_table_options,omitempty"` + ResultTableOptions metadata.ResultTableOptions `json:"-"` + // Scroll + Scroll string `json:"-"` // HighLight 是否打开高亮，只对原始数据接口生效 HighLight *metadata.HighLight `json:"highlight,omitempty"` @@ -911,11 +912,13 @@ func (q *Query) BuildMetadataQuery( query.QueryString = q.QueryString query.Source = q.KeepColumns query.HighLight = q.HighLight - query.Size = q.Limit - query.From = q.From + query.Scroll = q.Scroll query.ResultTableOptions = q.ResultTableOptions + query.Size = q.Limit + query.From = q.From + if len(allCondition) > 0 { query.AllConditions = make(metadata.AllConditions, len(allCondition)) for i, conditions := range allCondition { diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index 723db0bb4..e8bca55e8 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -11,6 +11,7 @@ package structured import ( "context" + "encoding/json" "testing" "time" @@ -270,7 +271,11 @@ func TestQueryToMetric(t *testing.T) { metric, err := c.query.ToQueryMetric(ctx, spaceUID) assert.Nil(t, err) - assert.Equal(t, c.metric, metric) + + a, _ := json.Marshal(c.metric) + b, _ := json.Marshal(metric) + + assert.JSONEq(t, string(a), string(b)) }) } } diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index a82accc30..b23fc58f5 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -14,7 +14,6 @@ import ( "fmt" "strings" "sync" - "sync/atomic" "time" ants "github.com/panjf2000/ants/v2" @@ -35,6 +34,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/prometheus" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/redis" ) @@ -175,27 +175,98 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot message strings.Builder lock sync.Mutex - querySize int64 + queryList []*metadata.Query ) + // 构建查询路由列表 + if queryTs.SpaceUid == "" { + queryTs.SpaceUid = metadata.GetUser(ctx).SpaceUid + } + for _, ql := range queryTs.QueryList { + // 时间复用 + ql.Timezone = queryTs.Timezone + ql.Start = queryTs.Start + ql.End = queryTs.End + + // 排序复用 + ql.OrderBy = queryTs.OrderBy + + // 如果 qry.Step 不存在去外部统一的 step + if ql.Step == "" { + ql.Step = queryTs.Step + } + + if queryTs.ResultTableOptions != nil { + ql.ResultTableOptions = queryTs.ResultTableOptions + } + + // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 + if ql.Limit == 0 && queryTs.Limit > 0 { + ql.Limit = queryTs.Limit + } + if ql.From == 0 && queryTs.From > 0 { + ql.From = queryTs.From + } + + // 复用 Scroll 配置 + if queryTs.Scroll != "" { + ql.Scroll = queryTs.Scroll + } + + // 复用 高亮配置，没有特殊配置的情况下使用公共配置 + if ql.HighLight != nil { + if !ql.HighLight.Enable && queryTs.HighLight.Enable { + ql.HighLight = queryTs.HighLight + } + } + + // 复用字段配置，没有特殊配置的情况下使用公共配置 + if len(ql.KeepColumns) == 0 && len(queryTs.ResultColumns) != 0 { + ql.KeepColumns = queryTs.ResultColumns + } + + qm, qmErr := ql.ToQueryMetric(ctx, queryTs.SpaceUid) + if qmErr != nil { + err = qmErr + return + } + + for _, qry := range qm.QueryList { + if qry != nil { + queryList = append(queryList, qry) + } + } + } + receiveWg.Add(1) - list = make([]map[string]any, 0) - resultTableOptions = make(metadata.ResultTableOptions) + resultTableOptions = queryTs.ResultTableOptions // 启动合并数据 go func() { defer receiveWg.Done() + for d := range dataCh { list = append(list, d) } - // dataCh close 之后会进入到该逻辑，多数据合并才需要进行排序和翻页 - if atomic.LoadInt64(&querySize) > 1 { - // 根据 orderby 对 list 进行排序 + if len(queryList) > 1 { queryTs.OrderBy.Orders().SortSliceList(list) - if queryTs.Limit > 0 && len(list) > queryTs.Limit { + // 判定是否启用 multi from 特性 + if resultTableOptions.IsMultiFrom() { list = list[0:queryTs.Limit] + + newResultTableOptions := metadata.ResultTableOptions{} + for _, l := range list { + if tableID, ok1 := l[elasticsearch.KeyTableID].(string); ok1 { + if address, ok2 := l[elasticsearch.KeyAddress].(string); ok2 { + opt := newResultTableOptions.GetOption(tableID, address) + opt.From += 1 + } + } + } + + resultTableOptions.MergeOptions(newResultTableOptions) } } }() @@ -208,83 +279,36 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot sendWg.Wait() close(dataCh) }() - - if queryTs.SpaceUid == "" { - queryTs.SpaceUid = metadata.GetUser(ctx).SpaceUid - } - for _, ql := range queryTs.QueryList { - // 时间复用 - ql.Timezone = queryTs.Timezone - ql.Start = queryTs.Start - ql.End = queryTs.End - - // 排序复用 - ql.OrderBy = queryTs.OrderBy - - // 如果 qry.Step 不存在去外部统一的 step - if ql.Step == "" { - ql.Step = queryTs.Step - } - - // 如果 Limit / From 没有单独指定的话，同时外部指定了的话，使用外部的 - if ql.Limit == 0 && queryTs.Limit > 0 { - ql.Limit = queryTs.Limit - } - if ql.From == 0 && queryTs.From > 0 { - ql.From = queryTs.From - } - - // 复用 Scroll 配置 - - // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if ql.HighLight != nil { - if !ql.HighLight.Enable && queryTs.HighLight.Enable { - ql.HighLight = queryTs.HighLight + for _, qry := range queryList { + sendWg.Add(1) + qry := qry + + // 如果是多数据合并，为了保证排序和Limit 的准确性，需要查询原始的所有数据，所以这里对 from 和 size 进行重写 + //if len(queryList) > 1 && qry.ResultTableOptions.IsMultiFrom() { + // qry.Size += qry.From + // qry.From = 0 + //} + + err = p.Submit(func() { + defer func() { + sendWg.Done() + }() + + instance := prometheus.GetTsDbInstance(ctx, qry) + if instance == nil { + log.Warnf(ctx, "not instance in %s", qry.StorageID) + return } - } - - // 复用字段配置，没有特殊配置的情况下使用公共配置 - if len(ql.KeepColumns) == 0 && len(queryTs.ResultColumns) != 0 { - ql.KeepColumns = queryTs.ResultColumns - } - qm, qmErr := ql.ToQueryMetric(ctx, queryTs.SpaceUid) - if qmErr != nil { - err = qmErr - return - } - - for _, qry := range qm.QueryList { - sendWg.Add(1) - qry := qry - err = p.Submit(func() { - defer func() { - sendWg.Done() - }() - - // 启动的查询数量 - atomic.AddInt64(&querySize, 1) - - instance := prometheus.GetTsDbInstance(ctx, qry) - if instance == nil { - log.Warnf(ctx, "not instance in %s", qry.StorageID) - return - } - - options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) - lock.Lock() - if queryErr != nil { - message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) - } else { - // 合并 ResultTableOptions - for k, v := range options { - resultTableOptions[k] = v - total += v.Total - } - } - lock.Unlock() - }) - } + options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) + lock.Lock() + if queryErr != nil { + message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) + } else { + resultTableOptions.MergeOptions(options) + } + lock.Unlock() + }) } }() @@ -294,6 +318,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot err = errors.New(message.String()) } + total = resultTableOptions.GetTotal() + return } diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 3847f1066..caf3a99f8 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -714,10 +714,12 @@ func TestQueryRawWithInstance(t *testing.T) { end := "1723595000" mock.Es.Set(map[string]any{ - `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, - `{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":20,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, - `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":1}`: `{"took":17,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"4f3a5e9c167097c9658e88b2f32364b2","_score":0.0,"_source":{"dtEventTimeStamp":"1723594209000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, - `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1723594000123,"include_lower":true,"include_upper":true,"to":1723595000234}}}}},"size":10}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, + `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, + `{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, + `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":1}`: `{"took":17,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"4f3a5e9c167097c9658e88b2f32364b2","_score":0.0,"_source":{"dtEventTimeStamp":"1723594209000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, + `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1723594000123,"include_lower":true,"include_upper":true,"to":1723595000234}}}}},"size":10}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, + //`{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":20,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, + //`{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, }) tcs := map[string]struct { @@ -742,7 +744,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: "1723595000234", }, total: 1e4, - expected: `[{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__doc_id":"c124dae69af9b86a7128ee4281820158","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"0826407be7f04f19086774ed68eac8dd"},{"__data_label":"bkbase_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"}]`, + expected: `[{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c124dae69af9b86a7128ee4281820158"},{"__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"dtEventTimeStamp":"1723594224000","__doc_id":"0826407be7f04f19086774ed68eac8dd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"},{"dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000"}]`, options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":11}}`, }, "query_bk_base_es_with_raw": { @@ -750,29 +752,31 @@ func TestQueryRawWithInstance(t *testing.T) { SpaceUid: spaceUid, QueryList: []*structured.Query{ { - DataSource: structured.BkLog, - TableID: structured.TableID(tableID), - From: 1, - Limit: 10, - KeepColumns: []string{"__ext.container_id", "dtEventTimeStamp"}, + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"__ext.container_id", "dtEventTimeStamp"}, + ReferenceName: "a", }, { - DataSource: structured.BkLog, - TableID: structured.TableID(tableID), - From: 20, - Limit: 10, - KeepColumns: []string{"__ext.io_kubernetes_pod", "dtEventTimeStamp"}, + DataSource: structured.BkLog, + TableID: structured.TableID(tableID), + KeepColumns: []string{"__ext.io_kubernetes_pod", "dtEventTimeStamp"}, + ReferenceName: "a", }, }, OrderBy: structured.OrderBy{ "-dtEventTimeStamp", "__doc_id", }, - Start: start, - End: end, + From: 10, + Limit: 10, + MetricMerge: "a", + Start: start, + End: end, }, total: 2e4, - expected: `[{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","dtEventTimeStamp":"1723594224000","__doc_id":"0826407be7f04f19086774ed68eac8dd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594224000"},{"__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c124dae69af9b86a7128ee4281820158","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000"},{"__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"328d487e284703b1d0bb8017dba46124","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000"},{"__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"8a916e558c71d4226f1d7f3279cf0fdd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"bd8a8ef60e94ade63c55c8773170d458","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000"},{"__doc_id":"c2dae031f095fa4b9deccf81964c7837","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"},{"_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"c726c895a380ba1a9df04ba4a977b29b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"c8401bb4ec021b038cb374593b8adce3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"dc888e9a3789976aa11483626fc61a4f"},{"__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"f6950fef394e813999d7316cdbf0de4d","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"fa209967d4a8c5d21b3e4f67d2cd579e","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","_time":"1723594161000","dtEventTimeStamp":"1723594161000"}]`, + expected: `[{"_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"0826407be7f04f19086774ed68eac8dd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c124dae69af9b86a7128ee4281820158","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000"}]`, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":10},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":10}}`, }, "query_bk_base_es_with_errors": { queryTs: &structured.QueryTs{ @@ -791,6 +795,7 @@ func TestQueryRawWithInstance(t *testing.T) { }, total: 1e4, expected: `[{"__data_label":"bkbase_es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, + options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":2}}`, }, } @@ -2500,7 +2505,7 @@ func TestQueryTsClusterMetrics(t *testing.T) { actual := string(out) assert.Nil(t, err) fmt.Printf("ActualResult: %v\n", actual) - assert.Equal(t, c.result, actual) + assert.JSONEq(t, c.result, actual) }) } } diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 09121052a..7f62ed94b 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -61,8 +61,10 @@ const ( KeyDocID = "__doc_id" KeyHighLight = "__highlight" - KeyIndex = "__index" - KeyTableID = "__result_table" + KeyIndex = "__index" + KeyTableID = "__result_table" + KeyAddress = "__address" + KeyDataLabel = "__data_label" ) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 0bbc2e928..5f5053977 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -303,7 +303,9 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac var res *elastic.SearchResult func() { - if opt, ok := qb.ResultTableOptions[qb.TableID+"|"+qo.conn.Address]; ok { + + if qb.ResultTableOptions != nil { + opt := qb.ResultTableOptions.GetOption(qb.TableID, qo.conn.Address) if opt.ScrollID != "" { res, err = client.Scroll(qo.indexes...).ScrollId(opt.ScrollID).Do(ctx) return @@ -547,7 +549,8 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star err error wg sync.WaitGroup - resultTableOptions = make(metadata.ResultTableOptions) + lock sync.Mutex + options = metadata.ResultTableOptions{} ) defer func() { @@ -563,20 +566,26 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star if query.DB == "" { err = fmt.Errorf("%s 查询别名为空", query.TableID) - return resultTableOptions, err + return nil, err } unit := metadata.GetQueryParams(ctx).TimeUnit aliases, err := i.getAlias(ctx, query.DB, query.NeedAddTime, start, end, query.Timezone) if err != nil { - return resultTableOptions, err + return nil, err } + errChan := make(chan error, len(i.connects)) + for _, conn := range i.connects { wg.Add(1) conn := conn go func() { - defer wg.Done() + defer func() { + errChan <- err + wg.Done() + }() + mappings, mappingErr := i.getMappings(ctx, conn, aliases) if mappingErr != nil { err = mappingErr @@ -591,6 +600,10 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star query.Size = i.maxSize } + if len(query.ResultTableOptions) > 0 { + query.From = query.ResultTableOptions.GetOption(query.TableID, conn.Address).From + } + qo := &queryOption{ indexes: aliases, start: start, @@ -611,7 +624,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } - option := metadata.ResultTableOption{} + option := &metadata.ResultTableOption{} if sr != nil { if sr.ScrollId != "" { option.ScrollID = sr.ScrollId @@ -630,6 +643,8 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star fact.data[KeyTableID] = query.TableID fact.data[KeyDataLabel] = query.DataLabel + fact.data[KeyAddress] = conn.Address + if timeValue, ok := data[fact.GetTimeField().Name]; ok { fact.data[FieldTime] = timeValue } @@ -640,20 +655,27 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star dataCh <- fact.data } - option.From = query.From + len(sr.Hits.Hits) - if sr.Hits.TotalHits != nil { option.Total = sr.Hits.TotalHits.Value } } } - resultTableOptions[query.TableID+"|"+conn.Address] = option + lock.Lock() + options.SetOption(query.TableID, conn.Address, option) + lock.Unlock() }() } wg.Wait() + close(errChan) + + for e := range errChan { + if e != nil { + return nil, e + } + } - return resultTableOptions, nil + return options, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 From 01ebf4d59829e0bfe58b5f211126e3aec6df3493 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 1 Apr 2025 11:25:41 +0800 Subject: [PATCH 042/104] =?UTF-8?q?feat:=20es=20=E8=81=9A=E5=90=88=20missi?= =?UTF-8?q?ng=20=E5=85=BC=E5=AE=B9=20int=20=E7=B1=BB=E5=9E=8B=E7=9A=84?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=20#1010158081122704186?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/format.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 09121052a..de30d0eed 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -588,7 +588,14 @@ func (f *FormatFactory) Agg() (name string, agg elastic.Aggregation, err error) name = info.Name case TermAgg: curName := info.Name - curAgg := elastic.NewTermsAggregation().Field(info.Name).Missing(" ") + curAgg := elastic.NewTermsAggregation().Field(info.Name) + fieldType, ok := f.mapping[info.Name] + if ok { + if fieldType == Text || fieldType == KeyWord { + curAgg = curAgg.Missing(" ") + } + } + if f.size > 0 { curAgg = curAgg.Size(f.size) } From 2328bf8708867716b99a423f59b4f8ab470d2395 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 1 Apr 2025 14:46:53 +0800 Subject: [PATCH 043/104] =?UTF-8?q?feat:=20es=20=E8=81=9A=E5=90=88=20missi?= =?UTF-8?q?ng=20=E5=85=BC=E5=AE=B9=20int=20=E7=B1=BB=E5=9E=8B=E7=9A=84?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=20#1010158081122704186?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/format.go | 6 +- .../tsdb/prometheus/merge_series_test.go | 201 ++++++++++++++++++ pkg/unify-query/tsdb/prometheus/querier.go | 1 + 3 files changed, 204 insertions(+), 4 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index de30d0eed..85dd15942 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -590,10 +590,8 @@ func (f *FormatFactory) Agg() (name string, agg elastic.Aggregation, err error) curName := info.Name curAgg := elastic.NewTermsAggregation().Field(info.Name) fieldType, ok := f.mapping[info.Name] - if ok { - if fieldType == Text || fieldType == KeyWord { - curAgg = curAgg.Missing(" ") - } + if !ok || fieldType == Text || fieldType == KeyWord { + curAgg = curAgg.Missing(" ") } if f.size > 0 { diff --git a/pkg/unify-query/tsdb/prometheus/merge_series_test.go b/pkg/unify-query/tsdb/prometheus/merge_series_test.go index 6a4cddb58..3bc8877cd 100644 --- a/pkg/unify-query/tsdb/prometheus/merge_series_test.go +++ b/pkg/unify-query/tsdb/prometheus/merge_series_test.go @@ -293,6 +293,207 @@ func TestMergeSeriesSet(t *testing.T) { }, fn: prometheus.NewMergeSeriesSetWithFuncAndSort(""), }, + "two queryResult with mergeSeriesSetWithFuncAndSort max": { + qrs: []*prompb.QueryResult{ + { + + Timeseries: []*prompb.TimeSeries{ + ts1, ts2, + }, + }, + { + + Timeseries: []*prompb.TimeSeries{ + ts3, ts4, + }, + }, + }, + ts: mock.TimeSeriesList{ + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "elasticsearch", + }, + }, + Samples: []prompb.Sample{ + { + Value: 8, + Timestamp: 60, + }, + { + Value: 9, + Timestamp: 120, + }, + }, + }, + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "prometheus", + }, + }, + Samples: []prompb.Sample{ + { + Value: 4, + Timestamp: 0, + }, + { + Value: 5, + Timestamp: 60, + }, + { + Value: 3, + Timestamp: 120, + }, + }, + }, + }, + fn: prometheus.NewMergeSeriesSetWithFuncAndSort("max"), + }, + "two queryResult with mergeSeriesSetWithFuncAndSort min": { + qrs: []*prompb.QueryResult{ + { + + Timeseries: []*prompb.TimeSeries{ + ts1, ts2, + }, + }, + { + + Timeseries: []*prompb.TimeSeries{ + ts3, ts4, + }, + }, + }, + ts: mock.TimeSeriesList{ + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "elasticsearch", + }, + }, + Samples: []prompb.Sample{ + { + Value: 2, + Timestamp: 60, + }, + { + Value: 3, + Timestamp: 120, + }, + }, + }, + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "prometheus", + }, + }, + Samples: []prompb.Sample{ + { + Value: 1, + Timestamp: 0, + }, + { + Value: 5, + Timestamp: 60, + }, + { + Value: 3, + Timestamp: 120, + }, + }, + }, + }, + fn: prometheus.NewMergeSeriesSetWithFuncAndSort("min"), + }, + "two queryResult with mergeSeriesSetWithFuncAndSort avg": { + qrs: []*prompb.QueryResult{ + { + + Timeseries: []*prompb.TimeSeries{ + ts1, ts2, + }, + }, + { + + Timeseries: []*prompb.TimeSeries{ + ts3, ts4, + }, + }, + }, + ts: mock.TimeSeriesList{ + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "elasticsearch", + }, + }, + Samples: []prompb.Sample{ + { + Value: 2, + Timestamp: 60, + }, + { + Value: 3, + Timestamp: 120, + }, + }, + }, + { + Labels: []prompb.Label{ + { + Name: "__name__", + Value: "up", + }, + { + Name: "job", + Value: "prometheus", + }, + }, + Samples: []prompb.Sample{ + { + Value: 1, + Timestamp: 0, + }, + { + Value: 5, + Timestamp: 60, + }, + { + Value: 3, + Timestamp: 120, + }, + }, + }, + }, + fn: prometheus.NewMergeSeriesSetWithFuncAndSort("avg"), + }, } for name, tc := range testCases { diff --git a/pkg/unify-query/tsdb/prometheus/querier.go b/pkg/unify-query/tsdb/prometheus/querier.go index b2550db99..99cdf087b 100644 --- a/pkg/unify-query/tsdb/prometheus/querier.go +++ b/pkg/unify-query/tsdb/prometheus/querier.go @@ -125,6 +125,7 @@ func (q *Querier) selectFn(hints *storage.SelectHints, matchers ...*labels.Match sets = append(sets, s) } } + set = storage.NewMergeSeriesSet(sets, NewMergeSeriesSetWithFuncAndSort(hints.Func)) }() From bc085c80d088f0fd3865af2f5ed662cb35376fe7 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 1 Apr 2025 15:14:48 +0800 Subject: [PATCH 044/104] feat: check_influxdb_group_by_metric_name #1010158081122720875 --- pkg/unify-query/metadata/query_reference.go | 5 ++++- pkg/unify-query/tsdb/influxdb/instance.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/metadata/query_reference.go b/pkg/unify-query/metadata/query_reference.go index d2febc3ae..fadebe452 100644 --- a/pkg/unify-query/metadata/query_reference.go +++ b/pkg/unify-query/metadata/query_reference.go @@ -55,7 +55,10 @@ func (q *Query) MetricLabels(ctx context.Context) *prompb.Label { encodeFunc = GetPromDataFormat(ctx).EncodeFunc() ) - metrics = append(metrics, q.DataSource) + if q.DataSource != "" { + metrics = append(metrics, q.DataSource) + } + for _, n := range strings.Split(q.TableID, ".") { metrics = append(metrics, n) } diff --git a/pkg/unify-query/tsdb/influxdb/instance.go b/pkg/unify-query/tsdb/influxdb/instance.go index e19d63505..54285f81b 100644 --- a/pkg/unify-query/tsdb/influxdb/instance.go +++ b/pkg/unify-query/tsdb/influxdb/instance.go @@ -633,7 +633,7 @@ func (i *Instance) grpcStream( span.Set("grpc-start-stream-series-set", name) - qry := &metadata.Query{TableID: fmt.Sprintf("%s.%s", db, measurement), Field: field} + qry := &metadata.Query{TableID: fmt.Sprintf("%s.%s", db, measurement), MetricName: field} seriesSet := StartStreamSeriesSet( ctx, name, &StreamSeriesSetOption{ From e79482fd5062073181ef49cb4fa526a303c9bf7d Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 2 Apr 2025 14:55:41 +0800 Subject: [PATCH 045/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/bkapi/bkapi_test.go | 6 +++--- pkg/unify-query/metadata/result_table_options.go | 9 ++++++++- pkg/unify-query/metadata/struct.go | 3 +++ pkg/unify-query/service/http/query.go | 13 ++++++++----- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/pkg/unify-query/bkapi/bkapi_test.go b/pkg/unify-query/bkapi/bkapi_test.go index d5db754cf..1d39a0b11 100644 --- a/pkg/unify-query/bkapi/bkapi_test.go +++ b/pkg/unify-query/bkapi/bkapi_test.go @@ -10,7 +10,7 @@ package bkapi import ( - "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -31,7 +31,7 @@ func TestGetBkAPI(t *testing.T) { "Content-Type": "application/json", }) - actual, _ := json.Marshal(headers) + actual := fmt.Sprintf("%s", headers) - assert.JSONEq(t, `{"Content-Type":"application/json","X-Bkapi-Authorization":"{\"bk_username\":\"admin\",\"bk_app_code\":\"bk_code\",\"bk_app_secret\":\"bk_secret\"}"}`, string(actual)) + assert.Equal(t, `map[Content-Type:application/json X-Bkapi-Authorization:{"bk_username":"admin","bk_app_code":"bk_code","bk_app_secret":"bk_secret"}]`, actual) } diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index 854e3b024..f3ae2a5b0 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -40,7 +40,10 @@ func (o ResultTableOptions) GetOption(tableID, address string) *ResultTableOptio if option, ok := o[tableID+"|"+address]; ok { return option } - return &ResultTableOption{} + return &ResultTableOption{ + From: 0, + Total: 0, + } } func (o ResultTableOptions) GetTotal() int64 { @@ -52,6 +55,10 @@ func (o ResultTableOptions) GetTotal() int64 { } func (o ResultTableOptions) IsMultiFrom() bool { + if len(o) == 0 { + return false + } + for _, v := range o { if v.ScrollID != "" || len(v.SearchAfter) > 0 { return false diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 1ae5fc2b2..45b85e1cc 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -357,6 +357,9 @@ func (os Orders) SortSliceList(list []map[string]any) { if len(os) == 0 { return } + if len(list) == 0 { + return + } sort.SliceStable(list, func(i, j int) bool { for _, o := range os { diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index b23fc58f5..71cf59f22 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -240,6 +240,9 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot receiveWg.Add(1) resultTableOptions = queryTs.ResultTableOptions + if resultTableOptions == nil { + resultTableOptions = make(metadata.ResultTableOptions) + } // 启动合并数据 go func() { @@ -253,7 +256,7 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot queryTs.OrderBy.Orders().SortSliceList(list) // 判定是否启用 multi from 特性 - if resultTableOptions.IsMultiFrom() { + if len(list) > queryTs.Limit && resultTableOptions.IsMultiFrom() { list = list[0:queryTs.Limit] newResultTableOptions := metadata.ResultTableOptions{} @@ -284,10 +287,10 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot qry := qry // 如果是多数据合并，为了保证排序和Limit 的准确性，需要查询原始的所有数据，所以这里对 from 和 size 进行重写 - //if len(queryList) > 1 && qry.ResultTableOptions.IsMultiFrom() { - // qry.Size += qry.From - // qry.From = 0 - //} + if len(queryList) > 1 && !qry.ResultTableOptions.IsMultiFrom() { + qry.Size += qry.From + qry.From = 0 + } err = p.Submit(func() { defer func() { From 2c2433778adb52ac1c779bdf3781b6143030b6d1 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 2 Apr 2025 15:15:57 +0800 Subject: [PATCH 046/104] =?UTF-8?q?feat:=20es=20=E5=85=A8=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E6=9F=A5=E8=AF=A2=E5=85=BC=E5=AE=B9=20#10101580811227?= =?UTF-8?q?50810?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/query_string.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index f9caeca63..34e101d66 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -43,7 +43,7 @@ func (s *QueryString) NestedFields() map[string]struct{} { } func (s *QueryString) queryString(str string) elastic.Query { - return elastic.NewQueryStringQuery(str).AnalyzeWildcard(true) + return elastic.NewQueryStringQuery(str).AnalyzeWildcard(true).Field("*").Field("__*").Lenient(true) } func (s *QueryString) Parser() (elastic.Query, error) { From 9bc072d299d9f172c9915dac242e39e9c3e13c1e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 3 Apr 2025 19:21:04 +0800 Subject: [PATCH 047/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/json/custom.go | 43 +++ .../metadata/result_table_options.go | 51 ++-- pkg/unify-query/mock/handler.go | 8 + pkg/unify-query/query/structured/query_ts.go | 2 + pkg/unify-query/service/http/query.go | 52 ++-- pkg/unify-query/service/http/query_test.go | 247 ++++++++++++++++-- pkg/unify-query/tsdb/bksql/instance.go | 4 +- pkg/unify-query/tsdb/elasticsearch/format.go | 8 +- .../tsdb/elasticsearch/format_test.go | 4 +- .../tsdb/elasticsearch/instance.go | 35 +-- .../tsdb/elasticsearch/instance_test.go | 8 +- pkg/unify-query/tsdb/influxdb/instance.go | 4 +- pkg/unify-query/tsdb/interfaces.go | 2 +- .../tsdb/offlineDataArchive/instance.go | 4 +- pkg/unify-query/tsdb/prometheus/instance.go | 4 +- .../tsdb/prometheus/merge_series_test.go | 67 ----- pkg/unify-query/tsdb/redis/instance.go | 4 +- .../tsdb/victoriaMetrics/instance.go | 4 +- 18 files changed, 357 insertions(+), 194 deletions(-) create mode 100644 pkg/unify-query/internal/json/custom.go diff --git a/pkg/unify-query/internal/json/custom.go b/pkg/unify-query/internal/json/custom.go new file mode 100644 index 000000000..01979ae13 --- /dev/null +++ b/pkg/unify-query/internal/json/custom.go @@ -0,0 +1,43 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package json + +import ( + "fmt" + "sort" + "strings" +) + +func MarshalListMap(data []map[string]interface{}) string { + if len(data) == 0 { + return "[]" + } + + var ( + s []string + ks []string + ) + for _, d := range data { + if len(ks) == 0 { + for k := range d { + ks = append(ks, k) + } + sort.Strings(ks) + } + + var m []string + for _, k := range ks { + m = append(m, fmt.Sprintf(`"%s":"%v"`, k, d[k])) + } + s = append(s, strings.Join(m, ",")) + } + + return fmt.Sprintf(`[{%s}]`, strings.Join(s, "},{")) +} diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index f3ae2a5b0..ba0b3191b 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -12,57 +12,42 @@ package metadata type ResultTableOptions map[string]*ResultTableOption type ResultTableOption struct { - Total int64 `json:"total,omitempty"` From int `json:"from,omitempty"` ScrollID string `json:"scroll_id,omitempty"` SearchAfter []any `json:"search_after,omitempty"` } -func (o ResultTableOptions) MergeOptions(options ResultTableOptions) { - for k, v := range options { - if s, ok := o[k]; ok { - v.Total += s.Total - v.From += s.From - - // ScrollID 和 SearchAfter 保持不变 - v.ScrollID = s.ScrollID - v.SearchAfter = s.SearchAfter - } - o[k] = v +func (o ResultTableOptions) FromInc(tableID, address string) { + option := o.GetOption(tableID, address) + if option != nil { + option.From++ } } func (o ResultTableOptions) SetOption(tableID, address string, option *ResultTableOption) { + if option == nil { + return + } + o[tableID+"|"+address] = option } func (o ResultTableOptions) GetOption(tableID, address string) *ResultTableOption { - if option, ok := o[tableID+"|"+address]; ok { - return option + if o == nil { + return &ResultTableOption{} } - return &ResultTableOption{ - From: 0, - Total: 0, - } -} -func (o ResultTableOptions) GetTotal() int64 { - var total int64 - for _, v := range o { - total += v.Total + if option, ok := o[tableID+"|"+address]; ok { + return option } - return total + return &ResultTableOption{} } -func (o ResultTableOptions) IsMultiFrom() bool { - if len(o) == 0 { - return false +func (o ResultTableOptions) MergeOptions(options ResultTableOptions) { + if o == nil { + return } - - for _, v := range o { - if v.ScrollID != "" || len(v.SearchAfter) > 0 { - return false - } + for k, v := range options { + o[k] = v } - return true } diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index 3da11b67e..d4a4d00c1 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -196,8 +196,16 @@ func mockElasticSearchHandler(ctx context.Context) { mappingResp := httpmock.NewStringResponder(http.StatusOK, mappings) httpmock.RegisterResponder(http.MethodGet, bkBaseUrl+"/es_index/_mapping/", mappingResp) httpmock.RegisterResponder(http.MethodGet, EsUrl+"/es_index/_mapping/", mappingResp) + httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search", searchHandler) + + httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search?scroll=5m", searchHandler) + httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search?scroll=5m", searchHandler) + + httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/_search/scroll", searchHandler) + httpmock.RegisterResponder(http.MethodPost, EsUrl+"/_search/scroll", searchHandler) + httpmock.RegisterResponder(http.MethodHead, EsUrl, searchHandler) } diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 6d0bcf84b..7a229de92 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -68,6 +68,8 @@ type QueryTs struct { // Scroll 是否启用 Scroll 查询 Scroll string `json:"scroll,omitempty"` + // IsMultiFrom 是否启用 MultiFrom 查询 + IsMultiFrom bool `json:"is_multi_from,omitempty"` ResultTableOptions metadata.ResultTableOptions `json:"result_table_options,omitempty"` diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 71cf59f22..db843d075 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -172,10 +172,9 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot receiveWg sync.WaitGroup dataCh = make(chan map[string]any) - message strings.Builder - lock sync.Mutex - + message strings.Builder queryList []*metadata.Query + lock sync.Mutex ) // 构建查询路由列表 @@ -255,27 +254,28 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot if len(queryList) > 1 { queryTs.OrderBy.Orders().SortSliceList(list) - // 判定是否启用 multi from 特性 - if len(list) > queryTs.Limit && resultTableOptions.IsMultiFrom() { - list = list[0:queryTs.Limit] - - newResultTableOptions := metadata.ResultTableOptions{} - for _, l := range list { - if tableID, ok1 := l[elasticsearch.KeyTableID].(string); ok1 { - if address, ok2 := l[elasticsearch.KeyAddress].(string); ok2 { - opt := newResultTableOptions.GetOption(tableID, address) - opt.From += 1 + // scroll 模式不进行裁剪 + if queryTs.Scroll == "" { + // 判定是否启用 multi from 特性 + if len(list) > queryTs.Limit { + if queryTs.IsMultiFrom { + list = list[0:queryTs.Limit] + for _, l := range list { + resultTableOptions.FromInc(l[elasticsearch.KeyTableID].(string), l[elasticsearch.KeyAddress].(string)) } + } else { + list = list[queryTs.From : queryTs.From+queryTs.Limit] } } - - resultTableOptions.MergeOptions(newResultTableOptions) } } }() // 多协程查询数据 - var sendWg sync.WaitGroup + var ( + sendWg sync.WaitGroup + ) + p, _ := ants.NewPool(QueryMaxRouting) go func() { defer func() { @@ -287,9 +287,11 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot qry := qry // 如果是多数据合并，为了保证排序和Limit 的准确性，需要查询原始的所有数据，所以这里对 from 和 size 进行重写 - if len(queryList) > 1 && !qry.ResultTableOptions.IsMultiFrom() { - qry.Size += qry.From - qry.From = 0 + if len(queryList) > 1 { + if !queryTs.IsMultiFrom { + qry.Size += qry.From + qry.From = 0 + } } err = p.Submit(func() { @@ -303,14 +305,16 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot return } - options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) - lock.Lock() + size, options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) if queryErr != nil { message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) - } else { - resultTableOptions.MergeOptions(options) } + + lock.Lock() + resultTableOptions.MergeOptions(options) lock.Unlock() + + total += size }) } }() @@ -321,8 +325,6 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot err = errors.New(message.String()) } - total = resultTableOptions.GetTotal() - return } diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index caf3a99f8..b7dfbff5b 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -30,6 +30,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/promql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" redisUtil "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/redis" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/redis" ) @@ -704,7 +705,6 @@ func TestQueryRawWithInstance(t *testing.T) { ctx := metadata.InitHashID(context.Background()) spaceUid := influxdb.SpaceUid - tableID := influxdb.ResultTableBkBaseEs mock.Init() influxdb.MockSpaceRouter(ctx) @@ -718,8 +718,15 @@ func TestQueryRawWithInstance(t *testing.T) { `{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":1}`: `{"took":17,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"4f3a5e9c167097c9658e88b2f32364b2","_score":0.0,"_source":{"dtEventTimeStamp":"1723594209000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1723594000123,"include_lower":true,"include_upper":true,"to":1723595000234}}}}},"size":10}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, - //`{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":20,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, - //`{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, + + // merge rt test mock data + `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}}]}}`, + `{"_source":{"includes":["a","b"]},"from":5,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, + `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}},{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, + + // scroll + `{"_source":{"includes":["a","b"]},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5,"sort":["_doc"]}`: `{"_scroll_id":"one","hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}}]}}`, + `{"scroll":"5m","scroll_id":"one"}`: `{"_scroll_id":"two","hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, }) tcs := map[string]struct { @@ -734,57 +741,166 @@ func TestQueryRawWithInstance(t *testing.T) { QueryList: []*structured.Query{ { DataSource: structured.BkLog, - TableID: structured.TableID(tableID), - From: 1, - Limit: 10, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), KeepColumns: []string{"__ext.container_id", "dtEventTimeStamp"}, }, }, + From: 1, + Limit: 10, Start: "1723594000123", End: "1723595000234", }, total: 1e4, - expected: `[{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c124dae69af9b86a7128ee4281820158"},{"__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"dtEventTimeStamp":"1723594224000","__doc_id":"0826407be7f04f19086774ed68eac8dd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"},{"dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000"}]`, - options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":11}}`, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"e058129ae18bff87c95e83f24584e654","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c124dae69af9b86a7128ee4281820158","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"43a36f412886bf30b0746562513638d3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"0826407be7f04f19086774ed68eac8dd","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"d56b4120194eb37f53410780da777d43","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"}]`, }, - "query_bk_base_es_with_raw": { + "query es with multi rt and multi from 0 - 5": { queryTs: &structured.QueryTs{ SpaceUid: spaceUid, QueryList: []*structured.Query{ { DataSource: structured.BkLog, TableID: structured.TableID(influxdb.ResultTableEs), - KeepColumns: []string{"__ext.container_id", "dtEventTimeStamp"}, + KeepColumns: []string{"a", "b"}, ReferenceName: "a", }, { DataSource: structured.BkLog, - TableID: structured.TableID(tableID), - KeepColumns: []string{"__ext.io_kubernetes_pod", "dtEventTimeStamp"}, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "a", + "b", + }, + Limit: 5, + MetricMerge: "a", + Start: start, + End: end, + IsMultiFrom: true, + ResultTableOptions: map[string]*metadata.ResultTableOption{ + "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + From: 0, + }, + "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { + From: 5, + }, + }, + }, + total: 246, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5}}`, + }, + "query es with multi rt and multi from 5 - 10": { + queryTs: &structured.QueryTs{ + SpaceUid: spaceUid, + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "a", + "b", + elasticsearch.KeyTableID, + }, + Limit: 5, + MetricMerge: "a", + Start: start, + End: end, + IsMultiFrom: true, + ResultTableOptions: map[string]*metadata.ResultTableOption{ + "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + From: 5, + }, + "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { + From: 5, + }, + }, + }, + total: 246, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":7},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":8}}`, + }, + "query es with multi rt and one from 0 - 5": { + queryTs: &structured.QueryTs{ + SpaceUid: spaceUid, + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "a", + "b", + elasticsearch.KeyTableID, + }, + From: 0, + Limit: 5, + MetricMerge: "a", + Start: start, + End: end, + }, + total: 246, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, + }, + "query es with multi rt and one from 5 - 10": { + queryTs: &structured.QueryTs{ + SpaceUid: spaceUid, + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, ReferenceName: "a", }, }, OrderBy: structured.OrderBy{ - "-dtEventTimeStamp", - "__doc_id", + "a", + "b", + elasticsearch.KeyTableID, }, - From: 10, - Limit: 10, + From: 5, + Limit: 5, MetricMerge: "a", Start: start, End: end, }, - total: 2e4, - expected: `[{"_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"0826407be7f04f19086774ed68eac8dd","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594224000","dtEventTimeStamp":"1723594224000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__doc_id":"d56b4120194eb37f53410780da777d43","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"218ceafd04f89b39cda7954e51f4a48a"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","dtEventTimeStamp":"1723594211000","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"43a36f412886bf30b0746562513638d3","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5"},{"_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es"},{"__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c124dae69af9b86a7128ee4281820158","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000","dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es"},{"dtEventTimeStamp":"1723594211000","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"e058129ae18bff87c95e83f24584e654","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","__data_label":"bkbase_es","__address":"http://127.0.0.1:12001/bk_data/query_sync/es","_time":"1723594211000"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":10},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":10}}`, + total: 246, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, }, - "query_bk_base_es_with_errors": { + "query_bk_base_es_1 to 1": { queryTs: &structured.QueryTs{ SpaceUid: spaceUid, QueryList: []*structured.Query{ { DataSource: structured.BkLog, - TableID: structured.TableID(tableID), + TableID: structured.TableID(influxdb.ResultTableEs), From: 1, Limit: 1, KeepColumns: []string{"__ext.container_id", "dtEventTimeStamp"}, @@ -794,8 +910,81 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 1e4, - expected: `[{"__data_label":"bkbase_es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, - options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"total":10000,"from":2}}`, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, + }, + "query with scroll - 1": { + queryTs: &structured.QueryTs{ + SpaceUid: spaceUid, + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "a", + "b", + elasticsearch.KeyTableID, + }, + From: 0, + Limit: 5, + MetricMerge: "a", + Start: start, + End: end, + Scroll: "5m", + }, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + total: 246, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"}}`, + }, + "query with scroll - 2": { + queryTs: &structured.QueryTs{ + SpaceUid: spaceUid, + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + { + DataSource: structured.BkLog, + TableID: structured.TableID(influxdb.ResultTableBkBaseEs), + KeepColumns: []string{"a", "b"}, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "a", + "b", + elasticsearch.KeyTableID, + }, + From: 0, + Limit: 5, + MetricMerge: "a", + Start: start, + End: end, + ResultTableOptions: map[string]*metadata.ResultTableOption{ + "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + ScrollID: "one", + }, + "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { + ScrollID: "one", + }, + }, + Scroll: "5m", + }, + expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"5"}]`, + total: 246, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"}}`, }, } @@ -808,11 +997,15 @@ func TestQueryRawWithInstance(t *testing.T) { } assert.Equal(t, c.total, total) - actual, _ := json.Marshal(list) - assert.JSONEq(t, c.expected, string(actual)) - optActual, _ := json.Marshal(options) - assert.JSONEq(t, c.options, string(optActual)) + actual := json.MarshalListMap(list) + + assert.Equal(t, c.expected, actual) + + if len(options) > 0 || c.options != "" { + optActual, _ := json.Marshal(options) + assert.JSONEq(t, c.options, string(optActual)) + } }) } } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 7d1befbcd..cbb80e46d 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -276,8 +276,8 @@ func (i *Instance) table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet { diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 3d9e1db39..6e771a717 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -177,7 +177,6 @@ type FormatFactory struct { aggInfoList aggInfoList orders metadata.Orders - from int size int timezone string @@ -253,7 +252,7 @@ func (f *FormatFactory) queryToUnix(t time.Time, unit string) int64 { } } -func (f *FormatFactory) WithQuery(valueKey string, timeField metadata.TimeField, start, end time.Time, timeFormat string, from, size int) *FormatFactory { +func (f *FormatFactory) WithQuery(valueKey string, timeField metadata.TimeField, start, end time.Time, timeFormat string, size int) *FormatFactory { if timeField.Name == "" { timeField.Name = DefaultTimeFieldName } @@ -272,7 +271,6 @@ func (f *FormatFactory) WithQuery(valueKey string, timeField metadata.TimeField, f.timeFormat = timeFormat f.valueField = valueKey f.timeField = timeField - f.from = from f.size = size return f @@ -673,10 +671,6 @@ func (f *FormatFactory) EsAgg(aggregates metadata.Aggregates) (string, elastic.A return f.Agg() } -func (f *FormatFactory) Size(ss *elastic.SearchSource) { - ss.From(f.from).Size(f.size) -} - func (f *FormatFactory) Orders() metadata.Orders { orders := make(metadata.Orders, 0, len(f.orders)) for _, order := range f.orders { diff --git a/pkg/unify-query/tsdb/elasticsearch/format_test.go b/pkg/unify-query/tsdb/elasticsearch/format_test.go index ebc3149cf..9862f9ff8 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/format_test.go @@ -381,7 +381,7 @@ func TestFormatFactory_RangeQueryAndAggregates(t *testing.T) { t.Run(name, func(t *testing.T) { ctx := metadata.InitHashID(context.Background()) fact := NewFormatFactory(ctx). - WithQuery("value", c.timeField, start, end, timeFormat, 0, 0). + WithQuery("value", c.timeField, start, end, timeFormat, 0). WithTransform(metadata.GetPromDataFormat(ctx).EncodeFunc(), metadata.GetPromDataFormat(ctx).DecodeFunc()) ss := elastic.NewSearchSource() @@ -432,7 +432,7 @@ func TestFormatFactory_AggDataFormat(t *testing.T) { Name: DefaultTimeFieldName, Type: DefaultTimeFieldType, Unit: DefaultTimeFieldUnit, - }, time.Time{}, time.Time{}, "", 0, 0) + }, time.Time{}, time.Time{}, "", 0) _, _, err := fact.EsAgg(c.aggregates) assert.NoError(t, err) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 5f5053977..47ddd2d04 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -266,7 +266,10 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac source.Size(0) source.Aggregation(name, agg) } else { - fact.Size(source) + source.Size(qb.Size) + if qb.Scroll == "" { + source.From(qb.From) + } } if qb.HighLight != nil && qb.HighLight.Enable { @@ -544,13 +547,13 @@ func (i *Instance) getAlias(ctx context.Context, db string, needAddTime bool, st } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { var ( err error wg sync.WaitGroup - lock sync.Mutex - options = metadata.ResultTableOptions{} + total int64 + resultTableOptions = make(metadata.ResultTableOptions) ) defer func() { @@ -566,13 +569,13 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star if query.DB == "" { err = fmt.Errorf("%s 查询别名为空", query.TableID) - return nil, err + return total, resultTableOptions, err } unit := metadata.GetQueryParams(ctx).TimeUnit aliases, err := i.getAlias(ctx, query.DB, query.NeedAddTime, start, end, query.Timezone) if err != nil { - return nil, err + return total, resultTableOptions, err } errChan := make(chan error, len(i.connects)) @@ -614,7 +617,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star fact := NewFormatFactory(ctx). WithIsReference(metadata.GetQueryParams(ctx).IsReference). - WithQuery(query.Field, query.TimeField, qo.start, qo.end, unit, query.From, query.Size). + WithQuery(query.Field, query.TimeField, qo.start, qo.end, unit, query.Size). WithMappings(mappings...). WithOrders(query.Orders) @@ -624,10 +627,12 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } - option := &metadata.ResultTableOption{} + var option *metadata.ResultTableOption if sr != nil { if sr.ScrollId != "" { - option.ScrollID = sr.ScrollId + option = &metadata.ResultTableOption{ + ScrollID: sr.ScrollId, + } } if sr.Hits != nil { @@ -656,14 +661,12 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } if sr.Hits.TotalHits != nil { - option.Total = sr.Hits.TotalHits.Value + total += sr.Hits.TotalHits.Value } } } - lock.Lock() - options.SetOption(query.TableID, conn.Address, option) - lock.Unlock() + resultTableOptions.SetOption(query.TableID, conn.Address, option) }() } wg.Wait() @@ -671,11 +674,11 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star for e := range errChan { if e != nil { - return nil, e + return total, resultTableOptions, e } } - return options, nil + return total, resultTableOptions, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 @@ -785,7 +788,7 @@ func (i *Instance) QuerySeriesSet( fact := NewFormatFactory(ctx). WithIsReference(metadata.GetQueryParams(ctx).IsReference). - WithQuery(query.Field, query.TimeField, qo.start, qo.end, unit, query.From, size). + WithQuery(query.Field, query.TimeField, qo.start, qo.end, unit, size). WithMappings(mappings...). WithOrders(query.Orders). WithTransform(metadata.GetPromDataFormat(ctx).EncodeFunc(), metadata.GetPromDataFormat(ctx).DecodeFunc()) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index fb1740a02..a344f1f5a 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -135,7 +135,7 @@ func TestInstance_queryReference(t *testing.T) { }, start: defaultStart, end: defaultEnd, - expected: `[{"__data_label":"es_index","__doc_id":"aS3KjpEBbwEm76LbcH1G","__highlight":{"group":["\u003cmark\u003efans\u003c/mark\u003e"],"user.first":["\u003cmark\u003eJohn\u003c/mark\u003e"],"user.new_first":["\u003cmark\u003eJohn\u003c/mark\u003e"],"user.new_group_user_first":["\u003cmark\u003eJohn\u003c/mark\u003e"]},"__index":"bk_unify_query_demo_2","__result_table":"es_index","group":"fans","user":[{"first":"John","last":"Smith"},{"first":"Alice","last":"White"}]}]`, + expected: `[{"__address":"http://127.0.0.1:93002","__highlight":{"group":["fans"],"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"]},"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans","__doc_id":"aS3KjpEBbwEm76LbcH1G","__index":"bk_unify_query_demo_2","__result_table":"es_index","__data_label":"es_index"}]`, }, "nested aggregate + query 测试": { query: &metadata.Query{ @@ -263,7 +263,7 @@ func TestInstance_queryReference(t *testing.T) { }, start: defaultStart, end: defaultEnd, - expected: `[{"__data_label":"set_10","__doc_id":"27bdd842c5f2929cf4bd90f1e4534a9d","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"d21cf5cf373b4a26a31774ff7ab38fad","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"e07e9f6437e64cc04e945dc0bf604e62","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"01fb133625637ee3b0b8e689b8126da2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"bcabf17aca864416784c0b1054b6056e","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"3edf7236b8fc45c1aec67ea68fa92c61","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"77d08d253f11554c5290b4cac515c4e1","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__doc_id":"573b3e1b4a499e4b7e7fab35f316ac8a","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"}]`, + expected: `[{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"27bdd842c5f2929cf4bd90f1e4534a9d","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"d21cf5cf373b4a26a31774ff7ab38fad","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"e07e9f6437e64cc04e945dc0bf604e62","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"01fb133625637ee3b0b8e689b8126da2","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"bcabf17aca864416784c0b1054b6056e","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"3edf7236b8fc45c1aec67ea68fa92c61"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"77d08d253f11554c5290b4cac515c4e1"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"573b3e1b4a499e4b7e7fab35f316ac8a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"}]`, }, "获取 10条 原始数据": { query: &metadata.Query{ @@ -291,7 +291,7 @@ func TestInstance_queryReference(t *testing.T) { }, start: defaultStart, end: defaultEnd, - expected: `[{"__data_label":"bk_log","__doc_id":"8defd23f1c2599e70f3ace3a042b2b5f","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"74ea55e7397582b101f0e21efbc876c6","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"084792484f943e314e31ef2b2e878115","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"0a3f47a7c57d0af7d40d82c729c37155","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"85981293cca7102b9560b49a7f089737","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"b429dc6611efafc4d02b90f882271dea","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"01213026ae064c6726fd99dc8276e842","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"93027432b40ccb01b1be8f4ea06a6853","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"bk_log","__doc_id":"bc31babcb5d1075fc421bd641199d3aa","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"}]`, + expected: `[{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8defd23f1c2599e70f3ace3a042b2b5f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__doc_id":"74ea55e7397582b101f0e21efbc876c6","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"084792484f943e314e31ef2b2e878115"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"0a3f47a7c57d0af7d40d82c729c37155","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"85981293cca7102b9560b49a7f089737","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"b429dc6611efafc4d02b90f882271dea","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__doc_id":"01213026ae064c6726fd99dc8276e842","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"93027432b40ccb01b1be8f4ea06a6853","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"bc31babcb5d1075fc421bd641199d3aa","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"}]`, }, "使用 promql 计算平均值 sum(count_over_time(field[12h]))": { query: &metadata.Query{ @@ -484,7 +484,7 @@ func TestInstance_queryReference(t *testing.T) { } }() - _, err = ins.QueryRawData(ctx, c.query, c.start, c.end, dataCh) + _, _, err = ins.QueryRawData(ctx, c.query, c.start, c.end, dataCh) close(dataCh) wg.Wait() diff --git a/pkg/unify-query/tsdb/influxdb/instance.go b/pkg/unify-query/tsdb/influxdb/instance.go index d7e39156b..6460dc42e 100644 --- a/pkg/unify-query/tsdb/influxdb/instance.go +++ b/pkg/unify-query/tsdb/influxdb/instance.go @@ -649,8 +649,8 @@ func (i *Instance) grpcStream( } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/interfaces.go b/pkg/unify-query/tsdb/interfaces.go index e8830a590..a1093c947 100644 --- a/pkg/unify-query/tsdb/interfaces.go +++ b/pkg/unify-query/tsdb/interfaces.go @@ -22,7 +22,7 @@ import ( ) type Instance interface { - QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) + QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) QuerySeriesSet(ctx context.Context, query *metadata.Query, start, end time.Time) storage.SeriesSet QueryExemplar(ctx context.Context, fields []string, query *metadata.Query, start, end time.Time, matchers ...*labels.Matcher) (*decoder.Response, error) diff --git a/pkg/unify-query/tsdb/offlineDataArchive/instance.go b/pkg/unify-query/tsdb/offlineDataArchive/instance.go index eeff6eb63..c50edf651 100644 --- a/pkg/unify-query/tsdb/offlineDataArchive/instance.go +++ b/pkg/unify-query/tsdb/offlineDataArchive/instance.go @@ -112,8 +112,8 @@ func (i Instance) setClient() error { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/prometheus/instance.go b/pkg/unify-query/tsdb/prometheus/instance.go index 0676e245e..f0354e0ee 100644 --- a/pkg/unify-query/tsdb/prometheus/instance.go +++ b/pkg/unify-query/tsdb/prometheus/instance.go @@ -69,8 +69,8 @@ func (i *Instance) InstanceType() string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 diff --git a/pkg/unify-query/tsdb/prometheus/merge_series_test.go b/pkg/unify-query/tsdb/prometheus/merge_series_test.go index 3bc8877cd..0e7a0d006 100644 --- a/pkg/unify-query/tsdb/prometheus/merge_series_test.go +++ b/pkg/unify-query/tsdb/prometheus/merge_series_test.go @@ -427,73 +427,6 @@ func TestMergeSeriesSet(t *testing.T) { }, fn: prometheus.NewMergeSeriesSetWithFuncAndSort("min"), }, - "two queryResult with mergeSeriesSetWithFuncAndSort avg": { - qrs: []*prompb.QueryResult{ - { - - Timeseries: []*prompb.TimeSeries{ - ts1, ts2, - }, - }, - { - - Timeseries: []*prompb.TimeSeries{ - ts3, ts4, - }, - }, - }, - ts: mock.TimeSeriesList{ - { - Labels: []prompb.Label{ - { - Name: "__name__", - Value: "up", - }, - { - Name: "job", - Value: "elasticsearch", - }, - }, - Samples: []prompb.Sample{ - { - Value: 2, - Timestamp: 60, - }, - { - Value: 3, - Timestamp: 120, - }, - }, - }, - { - Labels: []prompb.Label{ - { - Name: "__name__", - Value: "up", - }, - { - Name: "job", - Value: "prometheus", - }, - }, - Samples: []prompb.Sample{ - { - Value: 1, - Timestamp: 0, - }, - { - Value: 5, - Timestamp: 60, - }, - { - Value: 3, - Timestamp: 120, - }, - }, - }, - }, - fn: prometheus.NewMergeSeriesSetWithFuncAndSort("avg"), - }, } for name, tc := range testCases { diff --git a/pkg/unify-query/tsdb/redis/instance.go b/pkg/unify-query/tsdb/redis/instance.go index 7ffab88cf..928efae75 100644 --- a/pkg/unify-query/tsdb/redis/instance.go +++ b/pkg/unify-query/tsdb/redis/instance.go @@ -43,8 +43,8 @@ type Instance struct { ClusterMetricPrefix string } -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } func (i *Instance) Check(ctx context.Context, promql string, start, end time.Time, step time.Duration) string { diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance.go b/pkg/unify-query/tsdb/victoriaMetrics/instance.go index 5bad2022b..2f8eb05b8 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance.go @@ -115,8 +115,8 @@ func (i *Instance) Check(ctx context.Context, q string, start, end time.Time, st } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (metadata.ResultTableOptions, error) { - return nil, nil +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (int64, metadata.ResultTableOptions, error) { + return 0, nil, nil } // QuerySeriesSet 给 PromEngine 提供查询接口 From 58da710712d7113f0077ab2d7771a8e3e2d58eac Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 7 Apr 2025 14:50:50 +0800 Subject: [PATCH 048/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance_test.go | 36 +++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index b2442e045..9add38bf0 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -157,6 +157,7 @@ func TestInstance_bkSql(t *testing.T) { end := time.UnixMilli(1718193555000) testCases := []struct { + name string start time.Time end time.Time query *metadata.Query @@ -164,6 +165,7 @@ func TestInstance_bkSql(t *testing.T) { expected string }{ { + name: "namespace in and aggregate count", query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", @@ -188,6 +190,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 15000 * 15000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (dtEventTimeStamp + 0) / 15000 * 15000 - 0 ORDER BY `_timestamp_` ASC", }, { + name: "conditions with or", query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", @@ -222,6 +225,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, { + name: "conditions with or and", query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Measurement: sqlExpr.Doris, @@ -279,6 +283,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, { + name: "conditions with or and like", query: &metadata.Query{ DB: "132_lol_new_login_queue_login_1min", Field: "login_rate", @@ -313,6 +318,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, { + name: "aggregate sum", query: &metadata.Query{ DB: "132_hander_opmon_avg", Field: "value", @@ -326,6 +332,28 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", }, { + name: "aggregate multi function", + query: &metadata.Query{ + DB: "132_hander_opmon_avg", + Field: "value", + Aggregates: metadata.Aggregates{ + { + Name: "sum", + Field: "value", + Window: time.Second * 15, + }, + { + Name: "count", + Field: "other", + Window: time.Hour, + }, + }, + }, + + expected: "", + }, + { + name: "query raw", query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -335,6 +363,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' LIMIT 5", }, { + name: "query raw with order desc", query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -349,6 +378,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `_timestamp_` DESC", }, { + name: "query aggregate count and dimensions", query: &metadata.Query{ DB: "100133_ieod_logsearch4_errorlog_p", Measurement: "doris", @@ -367,6 +397,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY `ip` LIMIT 5", }, { + name: "query aggregate count", start: time.Unix(1733756400, 0), end: time.Unix(1733846399, 0), query: &metadata.Query{ @@ -382,6 +413,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", }, { + name: "query aggregate count with window hour", start: time.Unix(1733756400, 0), end: time.Unix(1733846399, 0), query: &metadata.Query{ @@ -399,8 +431,8 @@ func TestInstance_bkSql(t *testing.T) { }, } - for i, c := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + for _, c := range testCases { + t.Run(c.name, func(t *testing.T) { ctx := metadata.InitHashID(context.Background()) if c.start.Unix() <= 0 { c.start = start From b477c7a046d25171353a552771ed364ef623ab99 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 7 Apr 2025 21:14:24 +0800 Subject: [PATCH 049/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/function/function.go | 4 + .../metadata/result_table_options.go | 6 +- pkg/unify-query/mock/handler.go | 2 +- pkg/unify-query/service/http/query_test.go | 9 +- pkg/unify-query/tsdb/elasticsearch/format.go | 1 + .../tsdb/elasticsearch/instance.go | 49 +- .../tsdb/elasticsearch/instance_test.go | 596 +++++++++++------- 7 files changed, 406 insertions(+), 261 deletions(-) diff --git a/pkg/unify-query/internal/function/function.go b/pkg/unify-query/internal/function/function.go index e769a5c01..13acd141f 100644 --- a/pkg/unify-query/internal/function/function.go +++ b/pkg/unify-query/internal/function/function.go @@ -107,6 +107,10 @@ func ParseTimestamp(s string) (f string, t time.Time, err error) { return } +func IntPoint(d int) *int { + return &d +} + // QueryTimestamp 将开始时间和结束时间的时间戳从 string 转换为 time.Time，根据长度判定单位 func QueryTimestamp(s, e string) (format string, start time.Time, end time.Time, err error) { var ( diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index ba0b3191b..e7b1617a4 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -12,7 +12,7 @@ package metadata type ResultTableOptions map[string]*ResultTableOption type ResultTableOption struct { - From int `json:"from,omitempty"` + From *int `json:"from,omitempty"` ScrollID string `json:"scroll_id,omitempty"` SearchAfter []any `json:"search_after,omitempty"` } @@ -20,7 +20,9 @@ type ResultTableOption struct { func (o ResultTableOptions) FromInc(tableID, address string) { option := o.GetOption(tableID, address) if option != nil { - option.From++ + if option.From != nil { + *option.From++ + } } } diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index d4a4d00c1..ec97d0335 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -192,7 +192,7 @@ func mockElasticSearchHandler(ctx context.Context) { return } - mappings := `{"es_index":{"mappings":{"properties":{"group":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` + mappings := `{"es_index":{"mappings":{"properties":{"group":{"type":"keyword"},"kibana_stats":{"properties":{"kibana":{"properties":{"name":{"type":"keyword"}}}}},"timestamp":{"type":"log"},"type":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` mappingResp := httpmock.NewStringResponder(http.StatusOK, mappings) httpmock.RegisterResponder(http.MethodGet, bkBaseUrl+"/es_index/_mapping/", mappingResp) httpmock.RegisterResponder(http.MethodGet, EsUrl+"/es_index/_mapping/", mappingResp) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index b7dfbff5b..a02713677 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -23,6 +23,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/featureFlag" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb/decoder" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" @@ -781,10 +782,10 @@ func TestQueryRawWithInstance(t *testing.T) { IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { - From: 0, + From: function.IntPoint(0), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { - From: 5, + From: function.IntPoint(5), }, }, }, @@ -821,10 +822,10 @@ func TestQueryRawWithInstance(t *testing.T) { IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { - From: 5, + From: function.IntPoint(5), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { - From: 5, + From: function.IntPoint(5), }, }, }, diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 6e771a717..2501c9289 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -60,6 +60,7 @@ const ( const ( KeyDocID = "__doc_id" KeyHighLight = "__highlight" + KeySort = "sort" KeyIndex = "__index" KeyTableID = "__result_table" diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 47ddd2d04..bb68ad983 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -306,15 +306,16 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac var res *elastic.SearchResult func() { - if qb.ResultTableOptions != nil { opt := qb.ResultTableOptions.GetOption(qb.TableID, qo.conn.Address) if opt.ScrollID != "" { - res, err = client.Scroll(qo.indexes...).ScrollId(opt.ScrollID).Do(ctx) + res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).ScrollId(opt.ScrollID).Do(ctx) return } if len(opt.SearchAfter) > 0 { + source.SearchAfter(opt.SearchAfter...) + res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) return } } @@ -553,7 +554,8 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star wg sync.WaitGroup total int64 - resultTableOptions = make(metadata.ResultTableOptions) + lock sync.Mutex + resultTableOptions metadata.ResultTableOptions ) defer func() { @@ -566,6 +568,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star defer span.End(&err) span.Set("instance-connects", i.connects.String()) + span.Set("instance-query-result-table-options", query.ResultTableOptions) if query.DB == "" { err = fmt.Errorf("%s 查询别名为空", query.TableID) @@ -580,6 +583,8 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star errChan := make(chan error, len(i.connects)) + span.Set("aliases", aliases) + for _, conn := range i.connects { wg.Add(1) conn := conn @@ -604,7 +609,9 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } if len(query.ResultTableOptions) > 0 { - query.From = query.ResultTableOptions.GetOption(query.TableID, conn.Address).From + if query.ResultTableOptions.GetOption(query.TableID, conn.Address).From != nil { + query.From = *query.ResultTableOptions.GetOption(query.TableID, conn.Address).From + } } qo := &queryOption{ @@ -627,16 +634,12 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } - var option *metadata.ResultTableOption - if sr != nil { - if sr.ScrollId != "" { - option = &metadata.ResultTableOption{ - ScrollID: sr.ScrollId, - } - } + var addOption bool + option := &metadata.ResultTableOption{} + if sr != nil { if sr.Hits != nil { - for _, d := range sr.Hits.Hits { + for idx, d := range sr.Hits.Hits { data := make(map[string]any) if err = json.Unmarshal(d.Source, &data); err != nil { return @@ -657,6 +660,12 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star if len(d.Highlight) > 0 { fact.data[KeyHighLight] = d.Highlight } + + if idx == len(sr.Hits.Hits)-1 && d.Sort != nil { + option.SearchAfter = d.Sort + addOption = true + } + dataCh <- fact.data } @@ -666,7 +675,21 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } } - resultTableOptions.SetOption(query.TableID, conn.Address, option) + if sr.ScrollId != "" { + addOption = true + option.ScrollID = sr.ScrollId + } + + if addOption { + if resultTableOptions == nil { + resultTableOptions = make(metadata.ResultTableOptions) + } + + lock.Lock() + resultTableOptions.SetOption(query.TableID, conn.Address, option) + lock.Unlock() + } + }() } wg.Wait() diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index a344f1f5a..c95987209 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -27,6 +27,112 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" ) +func TestInstance_getAlias(t *testing.T) { + metadata.InitMetadata() + ctx := metadata.InitHashID(context.Background()) + inst, err := NewInstance(ctx, &InstanceOption{ + Connects: []Connect{ + { + Address: mock.EsUrl, + }, + }, + Timeout: time.Minute, + }) + if err != nil { + log.Panicf(ctx, err.Error()) + } + + for name, c := range map[string]struct { + start time.Time + end time.Time + timezone string + db string + needAddTime bool + + expected []string + }{ + "3d with UTC": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 3, 20, 0, 0, 0, time.UTC), + needAddTime: true, + expected: []string{"db_test_20240101*", "db_test_20240102*", "db_test_20240103*"}, + }, + "change month with Asia/ShangHai": { + start: time.Date(2024, 1, 25, 7, 10, 5, 0, time.UTC), + end: time.Date(2024, 2, 2, 6, 1, 4, 10, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_20240125*", "db_test_20240126*", "db_test_20240127*", "db_test_20240128*", "db_test_20240129*", "db_test_20240130*", "db_test_20240131*", "db_test_20240201*", "db_test_20240202*"}, + }, + "2d with Asia/ShangHai": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 3, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_20240102*", "db_test_20240103*", "db_test_20240104*"}, + }, + "14d with Asia/ShangHai": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 15, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_20240102*", "db_test_20240103*", "db_test_20240104*", "db_test_20240105*", "db_test_20240106*", "db_test_20240107*", "db_test_20240108*", "db_test_20240109*", "db_test_20240110*", "db_test_20240111*", "db_test_20240112*", "db_test_20240113*", "db_test_20240114*", "db_test_20240115*", "db_test_20240116*"}, + }, + "16d with Asia/ShangHai": { + start: time.Date(2024, 1, 15, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 2, 10, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_202401*", "db_test_202402*"}, + }, + "15d with Asia/ShangHai": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 16, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_202401*"}, + }, + "6m with Asia/ShangHai": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 7, 1, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_202401*", "db_test_202402*", "db_test_202403*", "db_test_202404*", "db_test_202405*", "db_test_202406*", "db_test_202407*"}, + }, + "7m with Asia/ShangHai": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 8, 1, 20, 0, 0, 0, time.UTC), + needAddTime: true, + timezone: "Asia/ShangHai", + expected: []string{"db_test_202402*", "db_test_202403*", "db_test_202404*", "db_test_202405*", "db_test_202406*", "db_test_202407*", "db_test_202408*"}, + }, + "2m and db": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 3, 1, 20, 0, 0, 0, time.UTC), + needAddTime: true, + db: "db_test,db_test_clone", + expected: []string{"db_test_202401*", "db_test_clone_202401*", "db_test_202402*", "db_test_clone_202402*", "db_test_202403*", "db_test_clone_202403*"}, + }, + "2m and db and not need add time": { + start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), + end: time.Date(2024, 3, 1, 20, 0, 0, 0, time.UTC), + needAddTime: false, + db: "db_test,db_test_clone", + expected: []string{"db_test", "db_test_clone"}, + }, + } { + t.Run(name, func(t *testing.T) { + if c.db == "" { + c.db = "db_test" + } + ctx = metadata.InitHashID(ctx) + actual, err := inst.getAlias(ctx, c.db, c.needAddTime, c.start, c.end, c.timezone) + assert.Nil(t, err) + assert.Equal(t, c.expected, actual) + }) + } +} + func TestInstance_queryReference(t *testing.T) { mock.Init() ctx := metadata.InitHashID(context.Background()) @@ -52,24 +158,12 @@ func TestInstance_queryReference(t *testing.T) { mock.Es.Set(map[string]any{ - // nested query + query string 测试 + highlight - `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, - - // "nested aggregate + query 测试 - `{"_source":{"includes":["group","user.first","user.last"]},"aggregations":{"user":{"aggregations":{"_value":{"value_count":{"field":"user.first"}}},"nested":{"path":"user"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":17,"relation":"eq"},"max_score":null,"hits":[]},"aggregations":{"user":{"doc_count":18,"_value":{"value":18}}}}`, - // 统计 __ext.io_kubernetes_pod 不为空的文档数量 `{"aggregations":{"_value":{"value_count":{"field":"__ext.io_kubernetes_pod"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":92,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":1523302}}}`, // 统计 __ext.io_kubernetes_pod 不为空的去重文档数量 `{"aggregations":{"_value":{"cardinality":{"field":"__ext.io_kubernetes_pod"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":170,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":4}}}`, - // 获取 10条 不 field 为空的原始数据 - `{"_source":{"includes":["__ext.container_id"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":13,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"27bdd842c5f2929cf4bd90f1e4534a9d","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d21cf5cf373b4a26a31774ff7ab38fad","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e07e9f6437e64cc04e945dc0bf604e62","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"01fb133625637ee3b0b8e689b8126da2","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bcabf17aca864416784c0b1054b6056e","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"3edf7236b8fc45c1aec67ea68fa92c61","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"77d08d253f11554c5290b4cac515c4e1","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"573b3e1b4a499e4b7e7fab35f316ac8a","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, - - // 获取 10条 原始数据 - `{"_source":{"includes":["__ext.io_kubernetes_pod","__ext.container_name"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8defd23f1c2599e70f3ace3a042b2b5f","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"74ea55e7397582b101f0e21efbc876c6","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"084792484f943e314e31ef2b2e878115","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0a3f47a7c57d0af7d40d82c729c37155","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"85981293cca7102b9560b49a7f089737","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"b429dc6611efafc4d02b90f882271dea","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"01213026ae064c6726fd99dc8276e842","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"93027432b40ccb01b1be8f4ea06a6853","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bc31babcb5d1075fc421bd641199d3aa","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}}]}}`, - // 使用 promql 计算平均值 sum(count_over_time(field[12h])) `{"aggregations":{"__ext.container_name":{"aggregations":{"__ext.io_kubernetes_pod":{"aggregations":{"dtEventTimeStamp":{"aggregations":{"_value":{"value_count":{"field":"dtEventTimeStamp"}}},"date_histogram":{"extended_bounds":{"max":1723679962000,"min":1723593608000},"field":"dtEventTimeStamp","interval":"12h","min_doc_count":0}}},"terms":{"field":"__ext.io_kubernetes_pod","missing":" "}}},"terms":{"field":"__ext.container_name","missing":" "}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":185,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"__ext.container_name":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"unify-query","doc_count":1523254,"__ext.io_kubernetes_pod":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"bkmonitor-unify-query-64bd4f5df4-599f9","doc_count":767743,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1723593600000","key":1723593600000,"doc_count":375064,"_value":{"value":375064}},{"key_as_string":"1723636800000","key":1723636800000,"doc_count":392679,"_value":{"value":392679}}]}},{"key":"bkmonitor-unify-query-64bd4f5df4-llp94","doc_count":755511,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1723593600000","key":1723593600000,"doc_count":381173,"_value":{"value":381173}},{"key_as_string":"1723636800000","key":1723636800000,"doc_count":374338,"_value":{"value":374338}}]}}]}},{"key":"sync-apigw","doc_count":48,"__ext.io_kubernetes_pod":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"bkmonitor-unify-query-apigw-sync-1178-cl8k8","doc_count":24,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1723593600000","key":1723593600000,"doc_count":24,"_value":{"value":24}},{"key_as_string":"1723636800000","key":1723636800000,"doc_count":0,"_value":{"value":0}}]}},{"key":"bkmonitor-unify-query-apigw-sync-1179-9h9xv","doc_count":24,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1723593600000","key":1723593600000,"doc_count":24,"_value":{"value":24}},{"key_as_string":"1723636800000","key":1723636800000,"doc_count":0,"_value":{"value":0}}]}}]}}]}}}`, @@ -87,8 +181,6 @@ func TestInstance_queryReference(t *testing.T) { // 根据 field 字段聚合 min，同时根据值排序 `{"aggregations":{"dtEventTimeStamp":{"aggregations":{"_value":{"min":{"field":"dtEventTimeStamp"}}},"terms":{"field":"dtEventTimeStamp","missing":" ","order":[{"_value":"asc"}]}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":198,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"dtEventTimeStamp":{"doc_count_error_upper_bound":-1,"sum_other_doc_count":1523292,"buckets":[{"key":1723593878000,"key_as_string":"1723593878000","doc_count":1,"_value":{"value":1}},{"key":1723593947000,"key_as_string":"1723593947000","doc_count":1,"_value":{"value":1}},{"key":1723594186000,"key_as_string":"1723594186000","doc_count":1,"_value":{"value":1}},{"key":1723595733000,"key_as_string":"1723595733000","doc_count":1,"_value":{"value":1}},{"key":1723596287000,"key_as_string":"1723596287000","doc_count":1,"_value":{"value":1}},{"key":1723596309000,"key_as_string":"1723596309000","doc_count":1,"_value":{"value":1}},{"key":1723596597000,"key_as_string":"1723596597000","doc_count":1,"_value":{"value":1}},{"key":1723596677000,"key_as_string":"1723596677000","doc_count":1,"_value":{"value":1}},{"key":1723596938000,"key_as_string":"1723596938000","doc_count":1,"_value":{"value":1}},{"key":1723597150000,"key_as_string":"1723597150000","doc_count":1,"_value":{"value":1}}]}}}`, - - `{"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"error":{"root_cause":[{"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"}],"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"},"status":400}`, }) for idx, c := range map[string]struct { @@ -101,42 +193,6 @@ func TestInstance_queryReference(t *testing.T) { expected string err error }{ - "nested query + query string 测试 + highlight": { - query: &metadata.Query{ - DB: db, - Field: "group", - From: 0, - Size: 5, - Orders: metadata.Orders{ - { - Name: FieldTime, - Ast: false, - }, - }, - DataSource: structured.BkLog, - TableID: "es_index", - DataLabel: "es_index", - MetricName: "group", - StorageType: consul.ElasticsearchStorageType, - Source: []string{"group", "user.first", "user.last"}, - AllConditions: metadata.AllConditions{ - { - { - DimensionName: "user.first", - Operator: "eq", - Value: []string{"John"}, - }, - }, - }, - QueryString: "group: fans", - HighLight: &metadata.HighLight{ - Enable: true, - }, - }, - start: defaultStart, - end: defaultEnd, - expected: `[{"__address":"http://127.0.0.1:93002","__highlight":{"group":["fans"],"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"]},"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans","__doc_id":"aS3KjpEBbwEm76LbcH1G","__index":"bk_unify_query_demo_2","__result_table":"es_index","__data_label":"es_index"}]`, - }, "nested aggregate + query 测试": { query: &metadata.Query{ DB: db, @@ -233,66 +289,6 @@ func TestInstance_queryReference(t *testing.T) { end: defaultEnd, expected: `[{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"}],"samples":[{"value":4,"timestamp":1723593608000}],"exemplars":null,"histograms":null}]`, }, - "获取 10条 不 field 为空的原始数据": { - query: &metadata.Query{ - DB: db, - Field: field, - From: 0, - Size: 10, - DataSource: structured.BkLog, - TableID: "bk_log_index_set_10", - DataLabel: "set_10", - MetricName: "__ext.io_kubernetes_pod", - Orders: metadata.Orders{ - { - Name: FieldTime, - Ast: false, - }, - }, - Source: []string{"__ext.container_id"}, - StorageType: consul.ElasticsearchStorageType, - AllConditions: metadata.AllConditions{ - { - { - DimensionName: field, - Operator: "ncontains", - Value: []string{""}, - }, - }, - }, - }, - start: defaultStart, - end: defaultEnd, - expected: `[{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"27bdd842c5f2929cf4bd90f1e4534a9d","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"d21cf5cf373b4a26a31774ff7ab38fad","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"e07e9f6437e64cc04e945dc0bf604e62","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"01fb133625637ee3b0b8e689b8126da2","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"bcabf17aca864416784c0b1054b6056e","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"3edf7236b8fc45c1aec67ea68fa92c61"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"77d08d253f11554c5290b4cac515c4e1"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"573b3e1b4a499e4b7e7fab35f316ac8a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"}]`, - }, - "获取 10条 原始数据": { - query: &metadata.Query{ - DB: db, - Field: field, - From: 0, - Size: 10, - Source: []string{"__ext.io_kubernetes_pod", "__ext.container_name"}, - DataSource: structured.BkLog, - TableID: "bk_log_index_set_10", - DataLabel: "bk_log", - MetricName: "__ext.io_kubernetes_pod", - StorageType: consul.ElasticsearchStorageType, - TimeField: metadata.TimeField{ - Name: "dtEventTimeStamp", - Type: TimeFieldTypeTime, - Unit: function.Millisecond, - }, - Orders: metadata.Orders{ - { - Name: FieldTime, - Ast: false, - }, - }, - }, - start: defaultStart, - end: defaultEnd, - expected: `[{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8defd23f1c2599e70f3ace3a042b2b5f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__doc_id":"74ea55e7397582b101f0e21efbc876c6","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"084792484f943e314e31ef2b2e878115"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"0a3f47a7c57d0af7d40d82c729c37155","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"85981293cca7102b9560b49a7f089737","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"b429dc6611efafc4d02b90f882271dea","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__doc_id":"01213026ae064c6726fd99dc8276e842","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"93027432b40ccb01b1be8f4ea06a6853","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"bc31babcb5d1075fc421bd641199d3aa","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"}]`, - }, "使用 promql 计算平均值 sum(count_over_time(field[12h]))": { query: &metadata.Query{ DB: db, @@ -446,165 +442,283 @@ func TestInstance_queryReference(t *testing.T) { end: defaultEnd, expected: `[{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723593878000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723593947000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723594186000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723595733000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723596287000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723596309000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723596597000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723596677000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723596938000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bklog:bk_log_index_set_10:__ext__bk_46__io_kubernetes_pod"},{"name":"dtEventTimeStamp","value":"1723597150000"}],"samples":[{"value":1,"timestamp":1723593608000}],"exemplars":null,"histograms":null}]`, }, - "query error with highlight max_analyzed_offset": { - query: &metadata.Query{ - DB: db, - Field: "error", - DataSource: structured.BkLog, - TableID: "check_error", - StorageType: consul.ElasticsearchStorageType, - }, - start: defaultStart, - end: defaultEnd, - err: fmt.Errorf("es query [es_index] error: [1:138] [highlight] unknown field [max_analyzed_offset]"), - }, } { t.Run(fmt.Sprintf("testing run: %s", idx), func(t *testing.T) { - if len(c.query.Aggregates) > 0 { - ss := ins.QuerySeriesSet(ctx, c.query, c.start, c.end) - timeSeries, err := mock.SeriesSetToTimeSeries(ss) - if err != nil { - log.Errorf(ctx, err.Error()) - return - } - - assert.JSONEq(t, c.expected, timeSeries.String()) - } else { - var ( - wg sync.WaitGroup - - list []any - ) - dataCh := make(chan map[string]any) - wg.Add(1) - go func() { - defer wg.Done() - for d := range dataCh { - list = append(list, d) - } - }() - - _, _, err = ins.QueryRawData(ctx, c.query, c.start, c.end, dataCh) - close(dataCh) - - wg.Wait() - - if c.err != nil { - assert.Equal(t, c.err, err) - } else { - assert.Nil(t, err) - res, _ := json.Marshal(list) - assert.JSONEq(t, c.expected, string(res)) - } - + ss := ins.QuerySeriesSet(ctx, c.query, c.start, c.end) + timeSeries, err := mock.SeriesSetToTimeSeries(ss) + if err != nil { + log.Errorf(ctx, err.Error()) + return } + assert.JSONEq(t, c.expected, timeSeries.String()) + }) } } -func TestInstance_getAlias(t *testing.T) { - metadata.InitMetadata() +func TestInstance_queryRawData(t *testing.T) { + mock.Init() ctx := metadata.InitHashID(context.Background()) - inst, err := NewInstance(ctx, &InstanceOption{ + + ins, err := NewInstance(ctx, &InstanceOption{ Connects: []Connect{ { Address: mock.EsUrl, }, }, - Timeout: time.Minute, + Timeout: 3 * time.Second, }) if err != nil { - log.Panicf(ctx, err.Error()) + t.Fatal(err) + return } - for name, c := range map[string]struct { - start time.Time - end time.Time - timezone string - db string - needAddTime bool + defaultStart := time.UnixMilli(1723593608000) + defaultEnd := time.UnixMilli(1723679962000) - expected []string + db := "es_index" + field := "dtEventTimeStamp" + + mock.Es.Set(map[string]any{ + + // nested query + query string 测试 + highlight + `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, + + // "nested aggregate + query 测试 + `{"_source":{"includes":["group","user.first","user.last"]},"aggregations":{"user":{"aggregations":{"_value":{"value_count":{"field":"user.first"}}},"nested":{"path":"user"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":17,"relation":"eq"},"max_score":null,"hits":[]},"aggregations":{"user":{"doc_count":18,"_value":{"value":18}}}}`, + + // 获取 10条 不 field 为空的原始数据 + `{"_source":{"includes":["__ext.container_id"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":13,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"27bdd842c5f2929cf4bd90f1e4534a9d","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d21cf5cf373b4a26a31774ff7ab38fad","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e07e9f6437e64cc04e945dc0bf604e62","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"01fb133625637ee3b0b8e689b8126da2","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bcabf17aca864416784c0b1054b6056e","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"3edf7236b8fc45c1aec67ea68fa92c61","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"77d08d253f11554c5290b4cac515c4e1","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"573b3e1b4a499e4b7e7fab35f316ac8a","_score":0.0,"_source":{"__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, + + // 获取 10条 原始数据 + `{"_source":{"includes":["__ext.io_kubernetes_pod","__ext.container_name"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":10,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8defd23f1c2599e70f3ace3a042b2b5f","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"74ea55e7397582b101f0e21efbc876c6","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"084792484f943e314e31ef2b2e878115","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0a3f47a7c57d0af7d40d82c729c37155","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"85981293cca7102b9560b49a7f089737","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"b429dc6611efafc4d02b90f882271dea","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"01213026ae064c6726fd99dc8276e842","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"93027432b40ccb01b1be8f4ea06a6853","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bc31babcb5d1075fc421bd641199d3aa","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}}]}}`, + + `{"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"error":{"root_cause":[{"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"}],"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"},"status":400}`, + + // scroll + `{"scroll":"10m","scroll_id":"scroll_id_1"}`: `{"_scroll_id":"scroll_id_1","took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8defd23f1c2599e70f3ace3a042b2b5f","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"74ea55e7397582b101f0e21efbc876c6","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"084792484f943e314e31ef2b2e878115","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0a3f47a7c57d0af7d40d82c729c37155","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}}]}}`, + + // search after + `{"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"search_after":[1743465646224,"kibana_settings",null],"size":5,"sort":[{"timestamp":{"order":"desc"}},{"type":{"order":"desc"}},{"kibana_stats.kibana.name":{"order":"desc"}}]}`: `{"took":13,"timed_out":false,"_shards":{"total":7,"successful":7,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[{"_index":".monitoring-kibana-7-2025.04.01","_id":"rYSm7pUBxj8-27WaYRCB","_score":null,"_source":{"timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465636224,"kibana_stats","es-os60crz7-kibana"]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"roSm7pUBxj8-27WaYRCB","_score":null,"_source":{"timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_settings"},"sort":[1743465636224,"kibana_settings",null]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"q4Sm7pUBxj8-27WaOhBx","_score":null,"_source":{"timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465626225,"kibana_stats","es-os60crz7-kibana"]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"rISm7pUBxj8-27WaOhBx","_score":null,"_source":{"timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_settings"},"sort":[1743465626225,"kibana_settings",null]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"8DSm7pUBipSLyy3IEwRg","_score":null,"_source":{"timestamp":"2025-04-01T00:00:16.224Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465616224,"kibana_stats","es-os60crz7-kibana"]}]}}`, + }) + + for idx, c := range map[string]struct { + query *metadata.Query + start time.Time + end time.Time + + isReference bool + + size int64 + list string + resultTableOptions metadata.ResultTableOptions + err error }{ - "3d with UTC": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 3, 20, 0, 0, 0, time.UTC), - needAddTime: true, - expected: []string{"db_test_20240101*", "db_test_20240102*", "db_test_20240103*"}, - }, - "change month with Asia/ShangHai": { - start: time.Date(2024, 1, 25, 7, 10, 5, 0, time.UTC), - end: time.Date(2024, 2, 2, 6, 1, 4, 10, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_20240125*", "db_test_20240126*", "db_test_20240127*", "db_test_20240128*", "db_test_20240129*", "db_test_20240130*", "db_test_20240131*", "db_test_20240201*", "db_test_20240202*"}, - }, - "2d with Asia/ShangHai": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 3, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_20240102*", "db_test_20240103*", "db_test_20240104*"}, - }, - "14d with Asia/ShangHai": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 15, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_20240102*", "db_test_20240103*", "db_test_20240104*", "db_test_20240105*", "db_test_20240106*", "db_test_20240107*", "db_test_20240108*", "db_test_20240109*", "db_test_20240110*", "db_test_20240111*", "db_test_20240112*", "db_test_20240113*", "db_test_20240114*", "db_test_20240115*", "db_test_20240116*"}, - }, - "16d with Asia/ShangHai": { - start: time.Date(2024, 1, 15, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 2, 10, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_202401*", "db_test_202402*"}, + "nested query + query string 测试 + highlight": { + query: &metadata.Query{ + DB: db, + Field: "group", + From: 0, + Size: 5, + Orders: metadata.Orders{ + { + Name: FieldTime, + Ast: false, + }, + }, + DataSource: structured.BkLog, + TableID: "es_index", + DataLabel: "es_index", + MetricName: "group", + StorageType: consul.ElasticsearchStorageType, + Source: []string{"group", "user.first", "user.last"}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "user.first", + Operator: "eq", + Value: []string{"John"}, + }, + }, + }, + QueryString: "group: fans", + HighLight: &metadata.HighLight{ + Enable: true, + }, + }, + start: defaultStart, + end: defaultEnd, + size: 1, + list: `[{"__address":"http://127.0.0.1:93002","__highlight":{"group":["fans"],"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"]},"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans","__doc_id":"aS3KjpEBbwEm76LbcH1G","__index":"bk_unify_query_demo_2","__result_table":"es_index","__data_label":"es_index"}]`, }, - "15d with Asia/ShangHai": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 1, 16, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_202401*"}, + "获取 10条 不 field 为空的原始数据": { + query: &metadata.Query{ + DB: db, + Field: field, + From: 0, + Size: 10, + DataSource: structured.BkLog, + TableID: "bk_log_index_set_10", + DataLabel: "set_10", + MetricName: "__ext.io_kubernetes_pod", + Orders: metadata.Orders{ + { + Name: FieldTime, + Ast: false, + }, + }, + Source: []string{"__ext.container_id"}, + StorageType: consul.ElasticsearchStorageType, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: field, + Operator: "ncontains", + Value: []string{""}, + }, + }, + }, + }, + start: defaultStart, + end: defaultEnd, + size: 1e4, + list: `[{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"27bdd842c5f2929cf4bd90f1e4534a9d","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"d21cf5cf373b4a26a31774ff7ab38fad","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"e07e9f6437e64cc04e945dc0bf604e62","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"01fb133625637ee3b0b8e689b8126da2","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"7eaa9e9edfc5e6bd8ba5df06fd2d5c00","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"bcabf17aca864416784c0b1054b6056e","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"3edf7236b8fc45c1aec67ea68fa92c61"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"77d08d253f11554c5290b4cac515c4e1"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"9fb5bb5f9bce7e0ab59e0cd1f410c57b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"},{"__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__doc_id":"573b3e1b4a499e4b7e7fab35f316ac8a","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"set_10","__address":"http://127.0.0.1:93002"}]`, }, - "6m with Asia/ShangHai": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 7, 1, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_202401*", "db_test_202402*", "db_test_202403*", "db_test_202404*", "db_test_202405*", "db_test_202406*", "db_test_202407*"}, + "获取 10条 原始数据": { + query: &metadata.Query{ + DB: db, + Field: field, + From: 0, + Size: 10, + Source: []string{"__ext.io_kubernetes_pod", "__ext.container_name"}, + DataSource: structured.BkLog, + TableID: "bk_log_index_set_10", + DataLabel: "bk_log", + MetricName: "__ext.io_kubernetes_pod", + StorageType: consul.ElasticsearchStorageType, + TimeField: metadata.TimeField{ + Name: "dtEventTimeStamp", + Type: TimeFieldTypeTime, + Unit: function.Millisecond, + }, + Orders: metadata.Orders{ + { + Name: FieldTime, + Ast: false, + }, + }, + }, + start: defaultStart, + end: defaultEnd, + size: 1e4, + list: `[{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8defd23f1c2599e70f3ace3a042b2b5f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__doc_id":"74ea55e7397582b101f0e21efbc876c6","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"084792484f943e314e31ef2b2e878115"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"0a3f47a7c57d0af7d40d82c729c37155","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__ext.container_name":"unify-query","__doc_id":"85981293cca7102b9560b49a7f089737","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"b429dc6611efafc4d02b90f882271dea","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__doc_id":"01213026ae064c6726fd99dc8276e842","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"93027432b40ccb01b1be8f4ea06a6853","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"bk_log","__address":"http://127.0.0.1:93002"},{"__data_label":"bk_log","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"bc31babcb5d1075fc421bd641199d3aa","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"}]`, }, - "7m with Asia/ShangHai": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 8, 1, 20, 0, 0, 0, time.UTC), - needAddTime: true, - timezone: "Asia/ShangHai", - expected: []string{"db_test_202402*", "db_test_202403*", "db_test_202404*", "db_test_202405*", "db_test_202406*", "db_test_202407*", "db_test_202408*"}, + "query error with highlight max_analyzed_offset": { + query: &metadata.Query{ + DB: db, + Field: "error", + DataSource: structured.BkLog, + TableID: "check_error", + StorageType: consul.ElasticsearchStorageType, + }, + start: defaultStart, + end: defaultEnd, + err: fmt.Errorf("es query [es_index] error: [1:138] [highlight] unknown field [max_analyzed_offset]"), }, - "2m and db": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 3, 1, 20, 0, 0, 0, time.UTC), - needAddTime: true, - db: "db_test,db_test_clone", - expected: []string{"db_test_202401*", "db_test_clone_202401*", "db_test_202402*", "db_test_clone_202402*", "db_test_202403*", "db_test_clone_202403*"}, + "query with scroll id": { + query: &metadata.Query{ + DB: db, + Field: field, + DataSource: structured.BkLog, + TableID: "bk_log_index_set_10", + StorageType: consul.ElasticsearchStorageType, + ResultTableOptions: metadata.ResultTableOptions{ + "bk_log_index_set_10|http://127.0.0.1:93002": &metadata.ResultTableOption{ + ScrollID: "scroll_id_1", + }, + }, + Scroll: "10m", + }, + start: defaultStart, + end: defaultEnd, + size: 1e4, + list: `[{"__data_label":"","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"8defd23f1c2599e70f3ace3a042b2b5f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10"},{"__doc_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"},{"__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"74ea55e7397582b101f0e21efbc876c6","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":""},{"__result_table":"bk_log_index_set_10","__data_label":"","__address":"http://127.0.0.1:93002","__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"084792484f943e314e31ef2b2e878115","__index":"v2_2_bklog_bk_unify_query_20240814_0"},{"__ext.container_name":"unify-query","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__doc_id":"0a3f47a7c57d0af7d40d82c729c37155","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"bk_log_index_set_10","__data_label":"","__address":"http://127.0.0.1:93002"}]`, + resultTableOptions: map[string]*metadata.ResultTableOption{ + "bk_log_index_set_10|http://127.0.0.1:93002": { + ScrollID: "scroll_id_1", + }, + }, }, - "2m and db and not need add time": { - start: time.Date(2024, 1, 1, 20, 0, 0, 0, time.UTC), - end: time.Date(2024, 3, 1, 20, 0, 0, 0, time.UTC), - needAddTime: false, - db: "db_test,db_test_clone", - expected: []string{"db_test", "db_test_clone"}, + "query with search after": { + query: &metadata.Query{ + DB: db, + Field: field, + DataSource: structured.BkLog, + TableID: "bk_log_index_set_10", + StorageType: consul.ElasticsearchStorageType, + Orders: []metadata.Order{ + { + Name: "timestamp", + Ast: false, + }, + { + Name: "type", + Ast: false, + }, + { + Name: "kibana_stats.kibana.name", + Ast: false, + }, + }, + Size: 5, + ResultTableOptions: metadata.ResultTableOptions{ + "bk_log_index_set_10|http://127.0.0.1:93002": &metadata.ResultTableOption{ + SearchAfter: []any{1743465646224, "kibana_settings", nil}, + }, + }, + }, + start: defaultStart, + end: defaultEnd, + size: 1e4, + list: `[{"__data_label":"","__address":"http://127.0.0.1:93002","timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_stats","kibana_stats.kibana.name":"es-os60crz7-kibana","__doc_id":"rYSm7pUBxj8-27WaYRCB","__index":".monitoring-kibana-7-2025.04.01","__result_table":"bk_log_index_set_10"},{"__address":"http://127.0.0.1:93002","timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_settings","__doc_id":"roSm7pUBxj8-27WaYRCB","__index":".monitoring-kibana-7-2025.04.01","__result_table":"bk_log_index_set_10","__data_label":""},{"__address":"http://127.0.0.1:93002","timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_stats","kibana_stats.kibana.name":"es-os60crz7-kibana","__doc_id":"q4Sm7pUBxj8-27WaOhBx","__index":".monitoring-kibana-7-2025.04.01","__result_table":"bk_log_index_set_10","__data_label":""},{"__address":"http://127.0.0.1:93002","timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_settings","__doc_id":"rISm7pUBxj8-27WaOhBx","__index":".monitoring-kibana-7-2025.04.01","__result_table":"bk_log_index_set_10","__data_label":""},{"__doc_id":"8DSm7pUBipSLyy3IEwRg","__index":".monitoring-kibana-7-2025.04.01","__result_table":"bk_log_index_set_10","__data_label":"","__address":"http://127.0.0.1:93002","timestamp":"2025-04-01T00:00:16.224Z","type":"kibana_stats","kibana_stats.kibana.name":"es-os60crz7-kibana"}]`, + resultTableOptions: map[string]*metadata.ResultTableOption{ + "bk_log_index_set_10|http://127.0.0.1:93002": { + SearchAfter: []any{1743465616224.0, "kibana_stats", "es-os60crz7-kibana"}, + }, + }, }, } { - t.Run(name, func(t *testing.T) { - if c.db == "" { - c.db = "db_test" + t.Run(fmt.Sprintf("testing run: %s", idx), func(t *testing.T) { + var ( + wg sync.WaitGroup + + list []any + ) + dataCh := make(chan map[string]any) + wg.Add(1) + go func() { + defer wg.Done() + for d := range dataCh { + list = append(list, d) + } + }() + + size, options, err := ins.QueryRawData(ctx, c.query, c.start, c.end, dataCh) + close(dataCh) + + wg.Wait() + + if c.err != nil { + assert.Equal(t, c.err, err) + } else { + assert.Nil(t, err) + res, _ := json.Marshal(list) + assert.JSONEq(t, c.list, string(res)) + + assert.Equal(t, c.size, size) + assert.Equal(t, c.resultTableOptions, options) } - ctx = metadata.InitHashID(ctx) - actual, err := inst.getAlias(ctx, c.db, c.needAddTime, c.start, c.end, c.timezone) - assert.Nil(t, err) - assert.Equal(t, c.expected, actual) }) } } From 94b374585d4a7bcdb6bd1cc6ad5e05940262e6b9 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 11:19:29 +0800 Subject: [PATCH 050/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/instance.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index bb68ad983..75fa51648 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -609,8 +609,11 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } if len(query.ResultTableOptions) > 0 { - if query.ResultTableOptions.GetOption(query.TableID, conn.Address).From != nil { - query.From = *query.ResultTableOptions.GetOption(query.TableID, conn.Address).From + option := query.ResultTableOptions.GetOption(query.TableID, conn.Address) + if option != nil { + if option.From != nil { + query.From = *option.From + } } } From 1dc09a1f89642d51cf8f75161da315e0b3e05aa4 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 11:44:41 +0800 Subject: [PATCH 051/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/instance.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 75fa51648..4f51b020d 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -309,11 +309,13 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac if qb.ResultTableOptions != nil { opt := qb.ResultTableOptions.GetOption(qb.TableID, qo.conn.Address) if opt.ScrollID != "" { + span.Set("query-scroll-id", opt.ScrollID) res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).ScrollId(opt.ScrollID).Do(ctx) return } if len(opt.SearchAfter) > 0 { + span.Set("query-search-after", opt.SearchAfter) source.SearchAfter(opt.SearchAfter...) res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) return @@ -321,8 +323,10 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac } if qb.Scroll != "" { + span.Set("query-scroll", qb.Scroll) res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).SearchSource(source).Do(ctx) } else { + span.Set("query-from", qb.From) res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) } }() From 1da26ba0919d68d11ed315cfd13a88bb2fb2801e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 11:51:03 +0800 Subject: [PATCH 052/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/metadata/result_table_options.go | 10 ++++++++++ pkg/unify-query/service/http/query.go | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index e7b1617a4..6bf686adf 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -53,3 +53,13 @@ func (o ResultTableOptions) MergeOptions(options ResultTableOptions) { o[k] = v } } + +// IsCrop 是否裁剪数据 +func (o ResultTableOptions) IsCrop() bool { + for _, v := range o { + if v.ScrollID != "" || len(v.SearchAfter) > 0 { + return false + } + } + return true +} diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index db843d075..e11b4ee29 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -251,12 +251,18 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot list = append(list, d) } + span.Set("query-list-num", len(queryList)) + if len(queryList) > 1 { queryTs.OrderBy.Orders().SortSliceList(list) - // scroll 模式不进行裁剪 - if queryTs.Scroll == "" { + span.Set("query-scroll", queryTs.Scroll) + span.Set("query-result-table", queryTs.ResultTableOptions) + + // scroll 和 searchAfter 模式不进行裁剪 + if queryTs.Scroll == "" || queryTs.ResultTableOptions.IsCrop() { // 判定是否启用 multi from 特性 + span.Set("query-multi-from", queryTs.IsMultiFrom) if len(list) > queryTs.Limit { if queryTs.IsMultiFrom { list = list[0:queryTs.Limit] From 4bcb683691f79b4d1cc6fca6b239642da54bf0aa Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 15:13:21 +0800 Subject: [PATCH 053/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata/result_table_options.go | 13 +----- pkg/unify-query/mock/handler.go | 2 +- pkg/unify-query/service/http/query.go | 34 +++++++++++---- pkg/unify-query/service/http/query_test.go | 14 ++++--- .../tsdb/elasticsearch/instance.go | 42 ++++++++++--------- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/pkg/unify-query/metadata/result_table_options.go b/pkg/unify-query/metadata/result_table_options.go index 6bf686adf..ac0743411 100644 --- a/pkg/unify-query/metadata/result_table_options.go +++ b/pkg/unify-query/metadata/result_table_options.go @@ -17,15 +17,6 @@ type ResultTableOption struct { SearchAfter []any `json:"search_after,omitempty"` } -func (o ResultTableOptions) FromInc(tableID, address string) { - option := o.GetOption(tableID, address) - if option != nil { - if option.From != nil { - *option.From++ - } - } -} - func (o ResultTableOptions) SetOption(tableID, address string, option *ResultTableOption) { if option == nil { return @@ -36,13 +27,13 @@ func (o ResultTableOptions) SetOption(tableID, address string, option *ResultTab func (o ResultTableOptions) GetOption(tableID, address string) *ResultTableOption { if o == nil { - return &ResultTableOption{} + return nil } if option, ok := o[tableID+"|"+address]; ok { return option } - return &ResultTableOption{} + return nil } func (o ResultTableOptions) MergeOptions(options ResultTableOptions) { diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index ec97d0335..4ccb2ec80 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -192,7 +192,7 @@ func mockElasticSearchHandler(ctx context.Context) { return } - mappings := `{"es_index":{"mappings":{"properties":{"group":{"type":"keyword"},"kibana_stats":{"properties":{"kibana":{"properties":{"name":{"type":"keyword"}}}}},"timestamp":{"type":"log"},"type":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` + mappings := `{"es_index":{"mappings":{"properties":{"a":{"type":"keyword"},"b":{"type":"keyword"},"group":{"type":"keyword"},"kibana_stats":{"properties":{"kibana":{"properties":{"name":{"type":"keyword"}}}}},"timestamp":{"type":"log"},"type":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` mappingResp := httpmock.NewStringResponder(http.StatusOK, mappings) httpmock.RegisterResponder(http.MethodGet, bkBaseUrl+"/es_index/_mapping/", mappingResp) httpmock.RegisterResponder(http.MethodGet, EsUrl+"/es_index/_mapping/", mappingResp) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index e11b4ee29..eb3ecd584 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -238,10 +238,6 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } receiveWg.Add(1) - resultTableOptions = queryTs.ResultTableOptions - if resultTableOptions == nil { - resultTableOptions = make(metadata.ResultTableOptions) - } // 启动合并数据 go func() { @@ -260,14 +256,28 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot span.Set("query-result-table", queryTs.ResultTableOptions) // scroll 和 searchAfter 模式不进行裁剪 - if queryTs.Scroll == "" || queryTs.ResultTableOptions.IsCrop() { + if queryTs.Scroll == "" && queryTs.ResultTableOptions.IsCrop() { // 判定是否启用 multi from 特性 span.Set("query-multi-from", queryTs.IsMultiFrom) if len(list) > queryTs.Limit { if queryTs.IsMultiFrom { + resultTableOptions = queryTs.ResultTableOptions + if resultTableOptions == nil { + resultTableOptions = make(metadata.ResultTableOptions) + } + list = list[0:queryTs.Limit] for _, l := range list { - resultTableOptions.FromInc(l[elasticsearch.KeyTableID].(string), l[elasticsearch.KeyAddress].(string)) + tableID := l[elasticsearch.KeyTableID].(string) + address := l[elasticsearch.KeyAddress].(string) + + option := resultTableOptions.GetOption(tableID, address) + if option == nil { + resultTableOptions.SetOption(tableID, address, &metadata.ResultTableOption{From: function.IntPoint(1)}) + } else { + *option.From++ + } + } } else { list = list[queryTs.From : queryTs.From+queryTs.Limit] @@ -316,9 +326,15 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) } - lock.Lock() - resultTableOptions.MergeOptions(options) - lock.Unlock() + // 如果配置了 IsMultiFrom，则无需使用 scroll 和 searchAfter 配置 + if !queryTs.IsMultiFrom { + if resultTableOptions == nil { + resultTableOptions = make(metadata.ResultTableOptions) + } + lock.Lock() + resultTableOptions.MergeOptions(options) + lock.Unlock() + } total += size }) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index a02713677..f83ca3903 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -721,12 +721,14 @@ func TestQueryRawWithInstance(t *testing.T) { `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":1,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1723594000123,"include_lower":true,"include_upper":true,"to":1723595000234}}}}},"size":10}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, // merge rt test mock data - `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}}]}}`, - `{"_source":{"includes":["a","b"]},"from":5,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, - `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}},{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, + `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5,"sort":[{"a":{"order":"asc"}},{"b":{"order":"asc"}}]}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"},"sort":["1","1"]},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"},"sort":["1","2"]},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"},"sort":["1","3"]},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"},"sort":["1","4"]},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"},"sort":["1","5"]}]}}`, + `{"_source":{"includes":["a","b"]},"from":5,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5,"sort":[{"a":{"order":"asc"}},{"b":{"order":"asc"}}]}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"},"sort":["2","1"]},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"},"sort":["2","2"]},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"},"sort":["2","3"]},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"},"sort":["2","4"]},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"},"sort":["2","5"]}]}}`, + `{"_source":{"includes":["a","b"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":10,"sort":[{"a":{"order":"asc"}},{"b":{"order":"asc"}}]}`: `{"hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"},"sort":["1","1"]},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"},"sort":["1","2"]},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"},"sort":["1","3"]},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"},"sort":["1","4"]},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"},"sort":["1","5"]},{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"},"sort":["2","1"]},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"},"sort":["2","2"]},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"},"sort":["2","3"]},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"},"sort":["2","4"]},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"},"sort":["2","5"]}]}}`, - // scroll - `{"_source":{"includes":["a","b"]},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5,"sort":["_doc"]}`: `{"_scroll_id":"one","hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}}]}}`, + // scroll with 5m + `{"_source":{"includes":["a","b"]},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":5,"sort":[{"a":{"order":"asc"}},{"b":{"order":"asc"}}]}`: `{"_scroll_id":"one","hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"1","_source":{"a":"1","b":"1"}},{"_index":"result_table_index","_id":"2","_source":{"a":"1","b":"2"}},{"_index":"result_table_index","_id":"3","_source":{"a":"1","b":"3"}},{"_index":"result_table_index","_id":"4","_source":{"a":"1","b":"4"}},{"_index":"result_table_index","_id":"5","_source":{"a":"1","b":"5"}}]}}`, + + // scroll id `{"scroll":"5m","scroll_id":"one"}`: `{"_scroll_id":"two","hits":{"total":{"value":123},"hits":[{"_index":"result_table_index","_id":"6","_source":{"a":"2","b":"1"}},{"_index":"result_table_index","_id":"7","_source":{"a":"2","b":"2"}},{"_index":"result_table_index","_id":"8","_source":{"a":"2","b":"3"}},{"_index":"result_table_index","_id":"9","_source":{"a":"2","b":"4"}},{"_index":"result_table_index","_id":"10","_source":{"a":"2","b":"5"}}]}}`, }) @@ -863,6 +865,7 @@ func TestQueryRawWithInstance(t *testing.T) { }, total: 246, expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]}}`, }, "query es with multi rt and one from 5 - 10": { queryTs: &structured.QueryTs{ @@ -894,6 +897,7 @@ func TestQueryRawWithInstance(t *testing.T) { }, total: 246, expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]},"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]}}`, }, "query_bk_base_es_1 to 1": { queryTs: &structured.QueryTs{ diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 4f51b020d..cd1dfbf57 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -308,18 +308,21 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac func() { if qb.ResultTableOptions != nil { opt := qb.ResultTableOptions.GetOption(qb.TableID, qo.conn.Address) - if opt.ScrollID != "" { - span.Set("query-scroll-id", opt.ScrollID) - res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).ScrollId(opt.ScrollID).Do(ctx) - return - } + if opt != nil { + if opt.ScrollID != "" { + span.Set("query-scroll-id", opt.ScrollID) + res, err = client.Scroll(qo.indexes...).Scroll(qb.Scroll).ScrollId(opt.ScrollID).Do(ctx) + return + } - if len(opt.SearchAfter) > 0 { - span.Set("query-search-after", opt.SearchAfter) - source.SearchAfter(opt.SearchAfter...) - res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) - return + if len(opt.SearchAfter) > 0 { + span.Set("query-search-after", opt.SearchAfter) + source.SearchAfter(opt.SearchAfter...) + res, err = client.Search().Index(qo.indexes...).SearchSource(source).Do(ctx) + return + } } + } if qb.Scroll != "" { @@ -641,8 +644,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } - var addOption bool - option := &metadata.ResultTableOption{} + var option *metadata.ResultTableOption if sr != nil { if sr.Hits != nil { @@ -669,8 +671,9 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } if idx == len(sr.Hits.Hits)-1 && d.Sort != nil { - option.SearchAfter = d.Sort - addOption = true + option = &metadata.ResultTableOption{ + SearchAfter: d.Sort, + } } dataCh <- fact.data @@ -682,21 +685,22 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } } + // ScrollID 覆盖 SearchAfter 配置 if sr.ScrollId != "" { - addOption = true - option.ScrollID = sr.ScrollId + option = &metadata.ResultTableOption{ + ScrollID: sr.ScrollId, + } } - if addOption { + if option != nil { if resultTableOptions == nil { - resultTableOptions = make(metadata.ResultTableOptions) + resultTableOptions = metadata.ResultTableOptions{} } lock.Lock() resultTableOptions.SetOption(query.TableID, conn.Address, option) lock.Unlock() } - }() } wg.Wait() From af38d63d059eb5348a0141cae04e6f5c7e64729c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 15:43:16 +0800 Subject: [PATCH 054/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/instance.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index cd1dfbf57..36b49776e 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -700,6 +700,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star lock.Lock() resultTableOptions.SetOption(query.TableID, conn.Address, option) lock.Unlock() + } }() } @@ -711,6 +712,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return total, resultTableOptions, e } } + span.Set("instance-out-result-table-options", resultTableOptions) return total, resultTableOptions, nil } From c12ea40e817e6e86b64ea590c90b57b127a912da Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 18:33:06 +0800 Subject: [PATCH 055/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tsdb/elasticsearch/instance.go | 12 ++++--- .../tsdb/elasticsearch/instance_test.go | 33 ++++++++++++++++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 36b49776e..3a3691389 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -357,6 +357,8 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac } return nil, errors.New(msg.String()) + } else if err.Error() == "EOF" { + return nil, nil } else { return nil, err } @@ -683,12 +685,12 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star total += sr.Hits.TotalHits.Value } } - } - // ScrollID 覆盖 SearchAfter 配置 - if sr.ScrollId != "" { - option = &metadata.ResultTableOption{ - ScrollID: sr.ScrollId, + // ScrollID 覆盖 SearchAfter 配置 + if sr.ScrollId != "" { + option = &metadata.ResultTableOption{ + ScrollID: sr.ScrollId, + } } } diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index c95987209..931f91cb4 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -496,9 +496,12 @@ func TestInstance_queryRawData(t *testing.T) { `{"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"error":{"root_cause":[{"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"}],"type":"x_content_parse_exception","reason":"[1:138] [highlight] unknown field [max_analyzed_offset]"},"status":400}`, - // scroll + // scroll_id_1 `{"scroll":"10m","scroll_id":"scroll_id_1"}`: `{"_scroll_id":"scroll_id_1","took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8defd23f1c2599e70f3ace3a042b2b5f","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"ba0a6e66f01d6cb77ae25b13ddf4ad1b","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"74ea55e7397582b101f0e21efbc876c6","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"084792484f943e314e31ef2b2e878115","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0a3f47a7c57d0af7d40d82c729c37155","_score":0.0,"_source":{"__ext":{"container_name":"unify-query","io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}}]}}`, + // scroll_id_2 + `{"scroll":"10m","scroll_id":"scroll_id_2"}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":0,"relation":"eq"},"max_score":null,"hits":[]}}`, + // search after `{"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"search_after":[1743465646224,"kibana_settings",null],"size":5,"sort":[{"timestamp":{"order":"desc"}},{"type":{"order":"desc"}},{"kibana_stats.kibana.name":{"order":"desc"}}]}`: `{"took":13,"timed_out":false,"_shards":{"total":7,"successful":7,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[{"_index":".monitoring-kibana-7-2025.04.01","_id":"rYSm7pUBxj8-27WaYRCB","_score":null,"_source":{"timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465636224,"kibana_stats","es-os60crz7-kibana"]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"roSm7pUBxj8-27WaYRCB","_score":null,"_source":{"timestamp":"2025-04-01T00:00:36.224Z","type":"kibana_settings"},"sort":[1743465636224,"kibana_settings",null]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"q4Sm7pUBxj8-27WaOhBx","_score":null,"_source":{"timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465626225,"kibana_stats","es-os60crz7-kibana"]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"rISm7pUBxj8-27WaOhBx","_score":null,"_source":{"timestamp":"2025-04-01T00:00:26.225Z","type":"kibana_settings"},"sort":[1743465626225,"kibana_settings",null]},{"_index":".monitoring-kibana-7-2025.04.01","_id":"8DSm7pUBipSLyy3IEwRg","_score":null,"_source":{"timestamp":"2025-04-01T00:00:16.224Z","type":"kibana_stats","kibana_stats":{"kibana":{"name":"es-os60crz7-kibana"}}},"sort":[1743465616224,"kibana_stats","es-os60crz7-kibana"]}]}}`, }) @@ -626,7 +629,7 @@ func TestInstance_queryRawData(t *testing.T) { end: defaultEnd, err: fmt.Errorf("es query [es_index] error: [1:138] [highlight] unknown field [max_analyzed_offset]"), }, - "query with scroll id": { + "query with scroll id 1": { query: &metadata.Query{ DB: db, Field: field, @@ -650,6 +653,24 @@ func TestInstance_queryRawData(t *testing.T) { }, }, }, + "query with scroll id 2": { + query: &metadata.Query{ + DB: db, + Field: field, + DataSource: structured.BkLog, + TableID: "bk_log_index_set_10", + StorageType: consul.ElasticsearchStorageType, + ResultTableOptions: metadata.ResultTableOptions{ + "bk_log_index_set_10|http://127.0.0.1:93002": &metadata.ResultTableOption{ + ScrollID: "scroll_id_2", + }, + }, + Scroll: "10m", + }, + start: defaultStart, + end: defaultEnd, + size: 0, + }, "query with search after": { query: &metadata.Query{ DB: db, @@ -713,8 +734,12 @@ func TestInstance_queryRawData(t *testing.T) { assert.Equal(t, c.err, err) } else { assert.Nil(t, err) - res, _ := json.Marshal(list) - assert.JSONEq(t, c.list, string(res)) + if len(list) > 0 { + res, _ := json.Marshal(list) + assert.JSONEq(t, c.list, string(res)) + } else { + assert.Nil(t, list) + } assert.Equal(t, c.size, size) assert.Equal(t, c.resultTableOptions, options) From ac38f267d1d5cf25db80490ea4c3eb8a1959b26c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 8 Apr 2025 18:42:30 +0800 Subject: [PATCH 056/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index f4402e91f..5ead655e3 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -246,6 +246,9 @@ func HandlerQueryRaw(c *gin.Context) { } return } + if listData.List == nil { + listData.List = make([]map[string]any, 0) + } resp.success(ctx, listData) } From 70f20b4d2e665a415d6379d6652fc56e2235ad47 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 10:52:18 +0800 Subject: [PATCH 057/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query.go | 10 ++++++++++ pkg/unify-query/tsdb/elasticsearch/instance.go | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index eb3ecd584..8d2cade50 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -203,6 +203,12 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot if ql.Limit == 0 && queryTs.Limit > 0 { ql.Limit = queryTs.Limit } + + // 在使用 multiFrom 模式下，From 需要保持为 0，因为 from 存放在 resultTableOptions 里面 + if queryTs.IsMultiFrom { + queryTs.From = 0 + } + if ql.From == 0 && queryTs.From > 0 { ql.From = queryTs.From } @@ -259,6 +265,10 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot if queryTs.Scroll == "" && queryTs.ResultTableOptions.IsCrop() { // 判定是否启用 multi from 特性 span.Set("query-multi-from", queryTs.IsMultiFrom) + span.Set("list-length", len(list)) + span.Set("query-ts-from", queryTs.From) + span.Set("query-ts-limit", queryTs.Limit) + if len(list) > queryTs.Limit { if queryTs.IsMultiFrom { resultTableOptions = queryTs.ResultTableOptions diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 3a3691389..03b36376c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -650,6 +650,9 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star if sr != nil { if sr.Hits != nil { + + span.Set("instance-out-list-size", len(sr.Hits.Hits)) + for idx, d := range sr.Hits.Hits { data := make(map[string]any) if err = json.Unmarshal(d.Source, &data); err != nil { @@ -714,6 +717,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return total, resultTableOptions, e } } + span.Set("instance-out-total", total) span.Set("instance-out-result-table-options", resultTableOptions) return total, resultTableOptions, nil From 44cc07c9bc636ac3d2170bcf7b4daa7f7d343d72 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 11:04:35 +0800 Subject: [PATCH 058/104] =?UTF-8?q?feat:=20es=E6=9F=A5=E8=AF=A2=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20scroll=20=E5=92=8C=20searchafter=20#101015808112261?= =?UTF-8?q?7468?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 8d2cade50..f77d81909 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -213,9 +213,10 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot ql.From = queryTs.From } - // 复用 Scroll 配置 + // 复用 scroll 配置，如果配置了 scroll 优先使用 scroll if queryTs.Scroll != "" { ql.Scroll = queryTs.Scroll + queryTs.IsMultiFrom = false } // 复用 高亮配置，没有特殊配置的情况下使用公共配置 From 2d5053e85058192446b1502ff941f619033cd0c0 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 12:15:23 +0800 Subject: [PATCH 059/104] =?UTF-8?q?feat:=20relation=20api=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=20#10101580811228?= =?UTF-8?q?52024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/cmdb/cmdb.go | 3 +- pkg/unify-query/cmdb/v1beta1/v1beta1.go | 7 +- pkg/unify-query/service/http/api/api.go | 97 ++++++++++++-------- pkg/unify-query/service/http/api/hook.go | 2 + pkg/unify-query/service/http/api/settings.go | 2 + pkg/unify-query/service/http/query.go | 2 + 6 files changed, 73 insertions(+), 40 deletions(-) diff --git a/pkg/unify-query/cmdb/cmdb.go b/pkg/unify-query/cmdb/cmdb.go index c3684cf7a..94069b142 100644 --- a/pkg/unify-query/cmdb/cmdb.go +++ b/pkg/unify-query/cmdb/cmdb.go @@ -11,7 +11,6 @@ package cmdb import ( "context" - "time" ) type CMDB interface { @@ -19,5 +18,5 @@ type CMDB interface { QueryResourceMatcher(ctx context.Context, lookBackDelta, spaceUid string, ts int64, target, source Resource, matcher Matcher, pathResource []Resource) (Resource, Matcher, []string, Matchers, error) // QueryResourceMatcherRange 获取目标的关键维度和值（query_range 查询） - QueryResourceMatcherRange(ctx context.Context, lookBackDelta, spaceUid string, step time.Duration, startTs, endTs int64, target, source Resource, matcher Matcher, pathResource []Resource) (Resource, Matcher, []string, []MatchersWithTimestamp, error) + QueryResourceMatcherRange(ctx context.Context, lookBackDelta, spaceUid string, step string, startTs, endTs int64, target, source Resource, matcher Matcher, pathResource []Resource) (Resource, Matcher, []string, []MatchersWithTimestamp, error) } diff --git a/pkg/unify-query/cmdb/v1beta1/v1beta1.go b/pkg/unify-query/cmdb/v1beta1/v1beta1.go index de2793609..91125fc71 100644 --- a/pkg/unify-query/cmdb/v1beta1/v1beta1.go +++ b/pkg/unify-query/cmdb/v1beta1/v1beta1.go @@ -357,12 +357,17 @@ func (r *model) QueryResourceMatcher(ctx context.Context, lookBackDelta, spaceUi return resource, matcher, path, shimMatcherWithTimestamp(ret), nil } -func (r *model) QueryResourceMatcherRange(ctx context.Context, lookBackDelta, spaceUid string, step time.Duration, startTs, endTs int64, target, source cmdb.Resource, matcher cmdb.Matcher, pathResource []cmdb.Resource) (cmdb.Resource, cmdb.Matcher, []string, []cmdb.MatchersWithTimestamp, error) { +func (r *model) QueryResourceMatcherRange(ctx context.Context, lookBackDelta, spaceUid string, stepString string, startTs, endTs int64, target, source cmdb.Resource, matcher cmdb.Matcher, pathResource []cmdb.Resource) (cmdb.Resource, cmdb.Matcher, []string, []cmdb.MatchersWithTimestamp, error) { unit, start, end, err := function.QueryTimestamp(strconv.FormatInt(startTs, 10), strconv.FormatInt(endTs, 10)) if err != nil { return "", nil, nil, nil, err } + step, err := time.ParseDuration(stepString) + if err != nil { + step = time.Minute + } + opt := QueryResourceOptions{ LookBackDelta: lookBackDelta, SpaceUid: spaceUid, diff --git a/pkg/unify-query/service/http/api/api.go b/pkg/unify-query/service/http/api/api.go index a7a55bb7c..06514f3fd 100644 --- a/pkg/unify-query/service/http/api/api.go +++ b/pkg/unify-query/service/http/api/api.go @@ -11,9 +11,10 @@ package api import ( "net/http" - "time" + "sync" "github.com/gin-gonic/gin" + ants "github.com/panjf2000/ants/v2" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/cmdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/cmdb/v1beta1" @@ -66,19 +67,36 @@ func HandlerAPIRelationMultiResource(c *gin.Context) { data := new(cmdb.RelationMultiResourceResponse) data.TraceID = span.TraceID() - data.Data = make([]cmdb.RelationMultiResourceResponseData, 0, len(request.QueryList)) - for _, qry := range request.QueryList { - d := cmdb.RelationMultiResourceResponseData{ - Code: http.StatusOK, - } + data.Data = make([]cmdb.RelationMultiResourceResponseData, len(request.QueryList)) - d.SourceType, d.SourceInfo, d.Path, d.TargetList, err = model.QueryResourceMatcher(ctx, qry.LookBackDelta, user.SpaceUid, qry.Timestamp, qry.TargetType, qry.SourceType, qry.SourceInfo, qry.PathResource) - if err != nil { - d.Message = err.Error() - d.Code = http.StatusBadRequest - } - data.Data = append(data.Data, d) + var ( + sendWg sync.WaitGroup + lock sync.Mutex + ) + p, _ := ants.NewPool(RelationMaxRouting) + defer p.Release() + + for idx, qry := range request.QueryList { + idx := idx + qry := qry + sendWg.Add(1) + _ = p.Submit(func() { + defer sendWg.Done() + d := cmdb.RelationMultiResourceResponseData{ + Code: http.StatusOK, + } + + d.SourceType, d.SourceInfo, d.Path, d.TargetList, err = model.QueryResourceMatcher(ctx, qry.LookBackDelta, user.SpaceUid, qry.Timestamp, qry.TargetType, qry.SourceType, qry.SourceInfo, qry.PathResource) + if err != nil { + d.Message = err.Error() + d.Code = http.StatusBadRequest + } + lock.Lock() + data.Data[idx] = d + lock.Unlock() + }) } + sendWg.Wait() resp.success(ctx, data) } @@ -126,33 +144,38 @@ func HandlerAPIRelationMultiResourceRange(c *gin.Context) { data := new(cmdb.RelationMultiResourceRangeResponse) data.TraceID = span.TraceID() - data.Data = make([]cmdb.RelationMultiResourceRangeResponseData, 0, len(request.QueryList)) - for _, qry := range request.QueryList { - d := cmdb.RelationMultiResourceRangeResponseData{ - Code: http.StatusOK, - } - - if qry.Step == "" { - qry.Step = "1m" - } + data.Data = make([]cmdb.RelationMultiResourceRangeResponseData, len(request.QueryList)) - step, err := time.ParseDuration(qry.Step) - if err != nil { - d.Message = err.Error() - d.Code = http.StatusBadRequest - data.Data = append(data.Data, d) - continue - } - - d.SourceType, d.SourceInfo, d.Path, d.TargetList, err = model.QueryResourceMatcherRange(ctx, qry.LookBackDelta, user.SpaceUid, step, qry.StartTs, qry.EndTs, qry.TargetType, qry.SourceType, qry.SourceInfo, qry.PathResource) - if err != nil { - log.Errorf(ctx, err.Error()) - - d.Message = err.Error() - d.Code = http.StatusBadRequest - } - data.Data = append(data.Data, d) + var ( + sendWg sync.WaitGroup + lock sync.Mutex + ) + p, _ := ants.NewPool(RelationMaxRouting) + defer p.Release() + + for idx, qry := range request.QueryList { + idx := idx + qry := qry + sendWg.Add(1) + _ = p.Submit(func() { + d := cmdb.RelationMultiResourceRangeResponseData{ + Code: http.StatusOK, + } + + d.SourceType, d.SourceInfo, d.Path, d.TargetList, err = model.QueryResourceMatcherRange(ctx, qry.LookBackDelta, user.SpaceUid, qry.Step, qry.StartTs, qry.EndTs, qry.TargetType, qry.SourceType, qry.SourceInfo, qry.PathResource) + if err != nil { + log.Errorf(ctx, err.Error()) + + d.Message = err.Error() + d.Code = http.StatusBadRequest + } + + lock.Lock() + data.Data[idx] = d + lock.Unlock() + }) } + sendWg.Wait() resp.success(ctx, data) } diff --git a/pkg/unify-query/service/http/api/hook.go b/pkg/unify-query/service/http/api/hook.go index fb6ff1f4b..2692ea764 100644 --- a/pkg/unify-query/service/http/api/hook.go +++ b/pkg/unify-query/service/http/api/hook.go @@ -20,11 +20,13 @@ import ( func setDefaultConfig() { viper.SetDefault(RelationMultiResourceConfigPath, "/api/v1/relation/multi_resource") viper.SetDefault(RelationMultiResourceRangeConfigPath, "/api/v1/relation/multi_resource_range") + viper.SetDefault(RelationMaxRoutingConfigPath, 5) } func loadConfig() { RelationMultiResource = viper.GetString(RelationMultiResourceConfigPath) RelationMultiResourceRange = viper.GetString(RelationMultiResourceRangeConfigPath) + RelationMaxRouting = viper.GetInt(RelationMaxRoutingConfigPath) } // init diff --git a/pkg/unify-query/service/http/api/settings.go b/pkg/unify-query/service/http/api/settings.go index b84974653..190942ba1 100644 --- a/pkg/unify-query/service/http/api/settings.go +++ b/pkg/unify-query/service/http/api/settings.go @@ -12,9 +12,11 @@ package api const ( RelationMultiResourceConfigPath = "api.relation.multi_resource" RelationMultiResourceRangeConfigPath = "api.relation.mutil_resource_range" + RelationMaxRoutingConfigPath = "api.relation.max_routing" ) var ( RelationMultiResource string RelationMultiResourceRange string + RelationMaxRouting int ) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 637acbe9f..bdacce295 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -201,6 +201,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot // 多协程查询数据 var sendWg sync.WaitGroup p, _ := ants.NewPool(QueryMaxRouting) + defer p.Release() + go func() { defer func() { sendWg.Wait() From af7ceff988c932805e442db08529e33471111292 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 14:32:55 +0800 Subject: [PATCH 060/104] =?UTF-8?q?feat:=20relation=20api=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=20#10101580811228?= =?UTF-8?q?52024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/api/api.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/unify-query/service/http/api/api.go b/pkg/unify-query/service/http/api/api.go index 06514f3fd..a94deb243 100644 --- a/pkg/unify-query/service/http/api/api.go +++ b/pkg/unify-query/service/http/api/api.go @@ -46,7 +46,7 @@ func HandlerAPIRelationMultiResource(c *gin.Context) { } ) - ctx, span := trace.NewSpan(ctx, "api-relation-multi-resource") + ctx, span := trace.NewSpan(ctx, "handler-api-relation-multi-resource") defer span.End(&err) request := new(cmdb.RelationMultiResourceRequest) @@ -57,7 +57,8 @@ func HandlerAPIRelationMultiResource(c *gin.Context) { } paramsBody, _ := json.Marshal(request) - span.Set("params-body", string(paramsBody)) + span.Set("handler-headers", c.Request.Header) + span.Set("handler-body", string(paramsBody)) model, err := v1beta1.GetModel(ctx) if err != nil { @@ -123,7 +124,7 @@ func HandlerAPIRelationMultiResourceRange(c *gin.Context) { } ) - ctx, span := trace.NewSpan(ctx, "api-relation-multi-resource-range") + ctx, span := trace.NewSpan(ctx, "handler-api-relation-multi-resource-range") defer span.End(&err) request := new(cmdb.RelationMultiResourceRangeRequest) @@ -134,7 +135,8 @@ func HandlerAPIRelationMultiResourceRange(c *gin.Context) { } paramsBody, _ := json.Marshal(request) - span.Set("params-body", string(paramsBody)) + span.Set("handler-headers", c.Request.Header) + span.Set("handler-body", string(paramsBody)) model, err := v1beta1.GetModel(ctx) if err != nil { From 04548cc81eba70ef9b193ac42835cfc0a138c0e7 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 15:12:02 +0800 Subject: [PATCH 061/104] =?UTF-8?q?feat:=20relation=20api=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=20#10101580811228?= =?UTF-8?q?52024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/bkapi/bkapi_test.go | 27 ++++++++++++++++--- pkg/unify-query/mock/handler.go | 2 ++ .../tsdb/elasticsearch/instance_test.go | 2 +- .../tsdb/elasticsearch/query_string_test.go | 16 +++++------ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/pkg/unify-query/bkapi/bkapi_test.go b/pkg/unify-query/bkapi/bkapi_test.go index 1d39a0b11..c7af41a5b 100644 --- a/pkg/unify-query/bkapi/bkapi_test.go +++ b/pkg/unify-query/bkapi/bkapi_test.go @@ -10,7 +10,7 @@ package bkapi import ( - "fmt" + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -31,7 +31,26 @@ func TestGetBkAPI(t *testing.T) { "Content-Type": "application/json", }) - actual := fmt.Sprintf("%s", headers) - - assert.Equal(t, `map[Content-Type:application/json X-Bkapi-Authorization:{"bk_username":"admin","bk_app_code":"bk_code","bk_app_secret":"bk_secret"}]`, actual) + actual := make(map[string]any) + for k, v := range headers { + var ( + nv map[string]string + ) + if err := json.Unmarshal([]byte(v), &nv); err != nil { + actual[k] = v + } else { + actual[k] = nv + } + } + + expected := map[string]any{ + "Content-Type": "application/json", + "X-Bkapi-Authorization": map[string]string{ + "bk_app_code": "bk_code", + "bk_app_secret": "bk_secret", + "bk_username": "admin", + }, + } + + assert.Equal(t, expected, actual) } diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index 4ccb2ec80..3aac1eb30 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -20,6 +20,7 @@ import ( "github.com/jarcoal/httpmock" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" ) type VmRequest struct { @@ -186,6 +187,7 @@ func mockElasticSearchHandler(ctx context.Context) { d, ok := Es.Get(string(body)) if !ok { err = fmt.Errorf(`es mock data is empty in "%s"`, body) + log.Errorf(ctx, err.Error()) return } w = httpmock.NewStringResponse(http.StatusOK, fmt.Sprintf("%s", d)) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index 931f91cb4..fd2c90d12 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -483,7 +483,7 @@ func TestInstance_queryRawData(t *testing.T) { mock.Es.Set(map[string]any{ // nested query + query string 测试 + highlight - `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, + `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"fields":["*","__*"],"lenient":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, // "nested aggregate + query 测试 `{"_source":{"includes":["group","user.first","user.last"]},"aggregations":{"user":{"aggregations":{"_value":{"value_count":{"field":"user.first"}}},"nested":{"path":"user"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":17,"relation":"eq"},"max_score":null,"hits":[]},"aggregations":{"user":{"doc_count":18,"_value":{"value":18}}}}`, diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index cc790c688..1311f3348 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -32,31 +32,31 @@ func TestQsToDsl(t *testing.T) { }{ { q: `log: "ERROR MSG"`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"log: \"ERROR MSG\""}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"log: \"ERROR MSG\""}}`, }, { q: `quick brown fox`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"quick brown fox"}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"quick brown fox"}}`, }, { q: `word.key: qu?ck`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"word.key: qu?ck"}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"word.key: qu?ck"}}`, }, { q: "\"message queue conflict\"", - expected: `{"query_string":{"analyze_wildcard":true,"query":"\"message queue conflict\""}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"\"message queue conflict\""}}`, }, { q: "\"message queue conflict\"", - expected: `{"query_string":{"analyze_wildcard":true,"query":"\"message queue conflict\""}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"\"message queue conflict\""}}`, }, { q: `nested.key: test AND demo`, - expected: `{"nested":{"path":"nested.key","query":{"bool":{"must":[{"match_phrase":{"nested.key":{"query":"test"}}},{"query_string":{"analyze_wildcard":true,"query":"\"demo\""}}]}}}}`, + expected: `{"nested":{"path":"nested.key","query":{"bool":{"must":[{"match_phrase":{"nested.key":{"query":"test"}}},{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"\"demo\""}}]}}}}`, }, { q: `sync_spaces AND -keyword AND -BKLOGAPI`, - expected: `{"query_string":{"analyze_wildcard":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, }, { q: `*`, @@ -66,7 +66,7 @@ func TestQsToDsl(t *testing.T) { }, { q: "ms: \u003e500 AND \"/fs-server\" AND NOT \"heartbeat\"", - expected: `{"query_string":{"analyze_wildcard":true,"query":"ms: \u003e500 AND \"/fs-server\" AND NOT \"heartbeat\""}}`, + expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"ms: \u003e500 AND \"/fs-server\" AND NOT \"heartbeat\""}}`, }, } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { From 51e24dca06d737de61d010c0a4008a11d6edbb02 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 15:55:36 +0800 Subject: [PATCH 062/104] =?UTF-8?q?feat:=20relation=20api=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=20#10101580811228?= =?UTF-8?q?52024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/api/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/unify-query/service/http/api/api.go b/pkg/unify-query/service/http/api/api.go index a94deb243..e786ed303 100644 --- a/pkg/unify-query/service/http/api/api.go +++ b/pkg/unify-query/service/http/api/api.go @@ -160,6 +160,7 @@ func HandlerAPIRelationMultiResourceRange(c *gin.Context) { qry := qry sendWg.Add(1) _ = p.Submit(func() { + defer sendWg.Done() d := cmdb.RelationMultiResourceRangeResponseData{ Code: http.StatusOK, } From ac6da6bef08c42b2ded69b3e77c2b87052adb18d Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 9 Apr 2025 16:25:09 +0800 Subject: [PATCH 063/104] =?UTF-8?q?feat:=20relation=20api=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=20#10101580811228?= =?UTF-8?q?52024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/cmdb/v1beta1/v1beta1.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/unify-query/cmdb/v1beta1/v1beta1.go b/pkg/unify-query/cmdb/v1beta1/v1beta1.go index 91125fc71..37d165daf 100644 --- a/pkg/unify-query/cmdb/v1beta1/v1beta1.go +++ b/pkg/unify-query/cmdb/v1beta1/v1beta1.go @@ -25,6 +25,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/cmdb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/promql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" @@ -290,7 +291,10 @@ func (r *model) queryResourceMatcher(ctx context.Context, opt QueryResourceOptio span.Set("paths", paths) metadata.GetQueryParams(ctx).SetTime(opt.Start, opt.End, opt.Unit).SetIsSkipK8s(true) - var errorMessage []string + var ( + errorMessage []string + ) + for _, path := range paths { reqTs, reqErr := r.doRequest(ctx, opt.LookBackDelta, opt.SpaceUid, opt.Start, opt.End, opt.Step, path, matcher, opt.Instant) if reqErr != nil { @@ -298,21 +302,14 @@ func (r *model) queryResourceMatcher(ctx context.Context, opt QueryResourceOptio continue } + hitPath = path if len(reqTs) > 0 { - hitPath = path ts = reqTs - span.Set("hit_path", hitPath) break } } - // 查询不到数据无需返回异常 - if len(ts) == 0 { - if len(errorMessage) > 0 { - err = errors.New(strings.Join(errorMessage, "\n")) - } - } - + span.Set("hit_path", hitPath) return } @@ -467,7 +464,8 @@ func (r *model) doRequest(ctx context.Context, lookBackDeltaStr, spaceUid string } if len(matrix) == 0 { - return nil, fmt.Errorf("instance data empty, promql: %s", realPromQL) + log.Warnf(ctx, "instance data empty, promql: %s", realPromQL) + return nil, nil } merged := make(map[int64]cmdb.Matchers) From a78152f1708aed37888f7bd3180b049609d8014d Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 11:41:29 +0800 Subject: [PATCH 064/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/Makefile | 4 + pkg/unify-query/influxdb/router_test.go | 477 ------------------ pkg/unify-query/kvstore/bbolt/bbolt.go | 4 + pkg/unify-query/tsdb/bksql/instance.go | 19 +- pkg/unify-query/tsdb/bksql/instance_test.go | 148 ++++-- .../tsdb/elasticsearch/query_string_test.go | 2 +- 6 files changed, 121 insertions(+), 533 deletions(-) delete mode 100644 pkg/unify-query/influxdb/router_test.go diff --git a/pkg/unify-query/Makefile b/pkg/unify-query/Makefile index 9e1723ec5..f5b1c5adc 100644 --- a/pkg/unify-query/Makefile +++ b/pkg/unify-query/Makefile @@ -50,3 +50,7 @@ addlicense: .PHONY: imports imports: addlicense goimports-reviser -rm-unused -set-alias -format -project-name "github.com/TencentBlueKing/bkmonitor-datalink/pkg" ./... + +.PHONY: race +race: + go test -race -timeout 1m -parallel 8 ./... \ No newline at end of file diff --git a/pkg/unify-query/influxdb/router_test.go b/pkg/unify-query/influxdb/router_test.go deleted file mode 100644 index 28b896fec..000000000 --- a/pkg/unify-query/influxdb/router_test.go +++ /dev/null @@ -1,477 +0,0 @@ -// Tencent is pleased to support the open source community by making -// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. -// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. -// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://opensource.org/licenses/MIT -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -package influxdb - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strings" - "sync" - "testing" - "time" - - "github.com/hashicorp/consul/api" - "github.com/influxdata/influxql" - "github.com/prashantv/gostub" - "github.com/stretchr/testify/assert" - - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" -) - -// FakeConsulRouter -func FakeConsulRouter(t *testing.T) *gostub.Stubs { - log.InitTestLogger() - _ = consul.SetInstance( - context.Background(), "", "test-unify", "http://127.0.0.1:8500", - []string{}, "127.0.0.1", 10205, "30s", "", "", "", - ) - consul.MetadataPath = "test/metadata/v1/default/data_id" - consul.MetricRouterPath = "test/metadata/influxdb_metrics" - - res := map[string]api.KVPairs{ - // dataid metadata - consul.MetadataPath: { - { - Key: consul.MetadataPath + "/1500009", - Value: []byte(`{"bk_data_id":1500009,"data_id":1500009,"mq_config":{"storage_config":{"topic":"0bkmonitor_15000090","partition":1},"cluster_config":{"domain_name":"kafka.service.consul","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","result_table_list":[{"bk_biz_id":2,"result_table":"process.port","shipper_list":[{"storage_config":{"real_table_name":"port","database":"process","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[{"field_name":"alive","type":"float","tag":"metric","default_value":"0","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_biz_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_collect_config_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_cloud_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_ip","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_service_category_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_topo_id","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"bk_target_topo_level","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"listen_address","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"listen_port","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"pid","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"process_name","type":"string","tag":"dimension","default_value":"","is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"target","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{}},{"field_name":"time","type":"timestamp","tag":"timestamp","default_value":"","is_config_by_user":true,"description":"\u6570\u636e\u4e0a\u62a5\u65f6\u95f4","unit":"","alias_name":"","option":{}}],"schema_type":"free","option":{}}],"option":{"inject_local_time":true,"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1500009/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"custom","token":"4774c8313d74430ca68c204aa6491eee","transfer_cluster_id":"default"}`), - }, - { - Key: consul.MetadataPath + "/1500015", - Value: []byte(`{"bk_biz_id":2,"bk_data_id":1500015,"data_id":1500015,"mq_config":{"storage_config":{"topic":"0bkmonitor_15000150","partition":1},"cluster_config":{"domain_name":"kafka.service.consul","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_event","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_event_public_1500015","shipper_list":[{"storage_config":{"index_datetime_format":"write_20060102","index_datetime_timezone":0,"date_format":"%Y%m%d","slice_size":500,"slice_gap":1440,"retention":30,"warm_phase_days":0,"warm_phase_settings":{},"base_index":"2_bkmonitor_event_public_1500015","index_settings":{"number_of_shards":4,"number_of_replicas":1},"mapping_settings":{"dynamic_templates":[{"discover_dimension":{"path_match":"dimensions.*","mapping":{"type":"keyword"}}}]}},"cluster_config":{"domain_name":"es7.service.consul","port":9200,"schema":null,"is_ssl_verify":false,"cluster_id":3,"cluster_name":"es7_cluster","version":"7.2","custom_option":"","registered_system":"_default","creator":"system","create_time":1624001652,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"elasticsearch","auth_info":{"password":"5gYTZqvd7Z7s","username":"elastic"}}],"field_list":[{"field_name":"dimensions","type":"object","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"object","es_dynamic":true}},{"field_name":"event","type":"object","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"object","es_properties":{"content":{"type":"text"},"count":{"type":"integer"}}}},{"field_name":"event_name","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"keyword"}},{"field_name":"target","type":"string","tag":"dimension","default_value":null,"is_config_by_user":true,"description":"","unit":"","alias_name":"","option":{"es_type":"keyword"}},{"field_name":"time","type":"timestamp","tag":"timestamp","default_value":"","is_config_by_user":true,"description":"\u6570\u636e\u4e0a\u62a5\u65f6\u95f4","unit":"","alias_name":"","option":{"es_type":"date_nanos","es_format":"epoch_millis"}}],"schema_type":"free","option":{"es_unique_field_list":["event","target","dimensions","event_name","time"]}}],"option":{"inject_local_time":true,"timestamp_precision":"ms","flat_batch_key":"data"},"type_label":"log","source_label":"bk_monitor","token":"d6dc05057e384f6db70e3542e3f8a2ce","transfer_cluster_id":"default"}`), - }, - { - Key: consul.MetadataPath + "/1573194", - Value: []byte(`{"bk_data_id":1573194,"data_id":1573194,"mq_config":{"storage_config":{"topic":"0bkmonitor_15731940","partition":7},"cluster_config":{"domain_name":"127.0.0.1","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","option":{"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1573194/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"bk_monitor","token":"9e180be8199946d3a8645639e236a50c","transfer_cluster_id":"default","data_name":"bcs_BCS-K8S-00000_k8s_metric","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_time_series_1573194.__default__","shipper_list":[{"storage_config":{"real_table_name":"__default__","database":"2_bkmonitor_time_series_1573194","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[],"schema_type":"free","option":{"is_split_measurement":true}}]}`), - }, - { - Key: consul.MetadataPath + "/1573195", - Value: []byte(`{"bk_data_id":1573195,"data_id":1573195,"mq_config":{"storage_config":{"topic":"0bkmonitor_15731950","partition":6},"cluster_config":{"domain_name":"127.0.0.1","port":9092,"schema":null,"is_ssl_verify":false,"cluster_id":1,"cluster_name":"kafka_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"kafka","auth_info":{"password":"","username":""}},"etl_config":"bk_standard_v2_time_series","option":{"timestamp_precision":"ms","flat_batch_key":"data","metrics_report_path":"bk_bkmonitorv3_enterprise_production/metadata/influxdb_metrics/1573195/time_series_metric","disable_metric_cutter":"true"},"type_label":"time_series","source_label":"bk_monitor","token":"f383f2f00f0449e7ba849fc8b6e28bad","transfer_cluster_id":"default","data_name":"bcs_BCS-K8S-00000_custom_metric","result_table_list":[{"bk_biz_id":2,"result_table":"2_bkmonitor_time_series_1573195.__default__","shipper_list":[{"storage_config":{"real_table_name":"__default__","database":"2_bkmonitor_time_series_1573195","retention_policy_name":""},"cluster_config":{"domain_name":"influxdb-proxy.bkmonitorv3.service.consul","port":10203,"schema":null,"is_ssl_verify":false,"cluster_id":2,"cluster_name":"influx_cluster1","version":null,"custom_option":"","registered_system":"_default","creator":"system","create_time":1574157128,"last_modify_user":"system","is_default_cluster":true},"cluster_type":"influxdb","auth_info":{"password":"","username":""}}],"field_list":[],"schema_type":"free","option":{"is_split_measurement":true}}]}`), - }, - }, - // metric info - consul.MetricRouterPath: { - { - Key: consul.MetricRouterPath + "/1500009/time_series_metric", - Value: []byte(`["alive"]`), - }, - { - Key: consul.MetricRouterPath + "/1573194/time_series_metric", - Value: []byte(`["node_memory_Cached_bytes", "node_filesystem_files"]`), - }, - { - Key: consul.MetricRouterPath + "/1573195/time_series_metric", - Value: []byte(`["node_memory_Cached_bytes", "node_filesystem_files"]`), - }, - }, - } - - consul.GetDataWithPrefix = func(p string) (api.KVPairs, error) { - return res[p], nil - } - consul.GetPathDataIDPath = func(metadataPath, version string) ([]string, error) { - return []string{metadataPath}, nil - } - stubs := gostub.New() - - _ = consul.ReloadBCSInfo() - reloadData, err := consul.ReloadRouterInfo() - assert.Nil(t, err) - ReloadTableInfos(reloadData) - metricData, err := consul.ReloadMetricInfo() - assert.Nil(t, err) - ReloadMetricRouter(metricData) - - return stubs -} - -// TestRouter_DBBiz -func TestRouter_DBBiz(t *testing.T) { - stubs := FakeConsulRouter(t) - defer stubs.Reset() - - expectsTsDBRouter := NewTsDBRouter() - expectsTsDBRouter.AddTables(1500009, []*consul.TableID{ - {DB: "process", Measurement: "port", IsSplitMeasurement: false, ClusterID: "2"}, - }) - expectsTsDBRouter.AddTables(1573194, []*consul.TableID{ - {DB: "2_bkmonitor_time_series_1573194", Measurement: "", IsSplitMeasurement: true, ClusterID: "2"}, - }) - expectsTsDBRouter.AddTables(1573195, []*consul.TableID{ - {DB: "2_bkmonitor_time_series_1573195", Measurement: "", IsSplitMeasurement: true, ClusterID: "2"}, - }) - - expectBizRouter := NewBizRouter() - expectBizRouter.AddRouter(2, []consul.DataID{ - 1500009, 1500015, 1573194, 1573195, - }...) - - expectMetricMap := NewMetricRouter() - expectMetricMap.AddRouter("alive", []consul.DataID{1500009}...) - expectMetricMap.AddRouter("node_memory_Cached_bytes", []consul.DataID{1573194, 1573195}...) - expectMetricMap.AddRouter("node_filesystem_files", []consul.DataID{1573194, 1573195}...) - - var dataIDs = []consul.DataID{1500009, 1500015, 1573194, 1573195} - var bizID = 2 - var metrics = []string{"alive", "node_memory_Cached_bytes", "node_filesystem_files"} - - assert.Equal(t, - expectsTsDBRouter.GetTableIDs(dataIDs...), - tsDBRouter.GetTableIDs(dataIDs...), - ) - assert.Equal(t, - expectBizRouter.GetRouter(bizID), - bizRouter.GetRouter(bizID), - ) - - assert.Equal(t, - expectMetricMap.GetRouter(metrics...), - metricRouter.GetRouter(metrics...), - ) -} - -const ( - ReferenceName = "m" - COUNT = "count" -) - -// ctxKeyPromQueryKeyName -type ctxKeyPromQueryKeyName struct { -} - -// ctxKeyPromMetricMappingKey -type ctxKeyPromMetricMappingKey struct { -} - -// QueryInfo -type QueryInfo struct { - DB string - Measurement string - DataIDList []consul.DataID - - // 是否为行转列表 - IsPivotTable bool -} - -// QueryInfoIntoContext -func QueryInfoIntoContext(ctx context.Context, metrics string, queryInfo *QueryInfo) context.Context { - var ( - buffer map[string]*QueryInfo - ok bool - ) - if buffer, ok = ctx.Value(&ctxKeyPromQueryKeyName{}).(map[string]*QueryInfo); !ok { - buffer = make(map[string]*QueryInfo) - } - buffer[metrics] = queryInfo - - ctx = context.WithValue(ctx, &ctxKeyPromQueryKeyName{}, buffer) - return ctx -} - -// MetricMappingIntoContext -func MetricMappingIntoContext(ctx context.Context, referenceName string, metricName string) context.Context { - var mapping map[string]string - var ok bool - if mapping, ok = ctx.Value(&ctxKeyPromMetricMappingKey{}).(map[string]string); !ok { - mapping = make(map[string]string) - } - mapping[referenceName] = metricName - return context.WithValue(ctx, &ctxKeyPromMetricMappingKey{}, mapping) -} - -// MetricMappingFromContext -func MetricMappingFromContext(ctx context.Context, referenceName string) (string, error) { - if mapping, ok := ctx.Value(&ctxKeyPromMetricMappingKey{}).(map[string]string); ok { - if metricName, ok := mapping[referenceName]; ok { - return metricName, nil - } - return "", errors.New("get metric mapping failed") - } - return "", errors.New("get metric mapping failed") -} - -// where类型说明，决定了where语句的渲染格式 -type ValueType int - -const ( - StringType1 ValueType = 0 - NumType ValueType = 1 - RegexpType ValueType = 2 - TextType ValueType = 3 -) - -// 操作符号映射 -type Operator string - -const ( - EqualOperator string = "=" - NEqualOperator string = "!=" - UpperOperator string = ">" - UpperEqualOperator string = ">=" - LowerOperator string = "<" - LowerEqualOperator string = "<=" - RegexpOperator string = "=~" - NRegexpOperator string = "!~" -) - -const ( - AndOperator string = "and" - OrOperator string = "or" -) - -// WhereList -type WhereList struct { - whereList []*Where - logicalList []string -} - -// NewWhereList -func NewWhereList() *WhereList { - return &WhereList{ - whereList: make([]*Where, 0), - logicalList: make([]string, 0), - } -} - -// Append -func (l *WhereList) Append(logicalOperator string, where *Where) { - l.logicalList = append(l.logicalList, logicalOperator) - l.whereList = append(l.whereList, where) -} - -// String -func (l *WhereList) String() string { - b := new(strings.Builder) - for index, where := range l.whereList { - if index == 0 { - b.WriteString("where ") - } else { - b.WriteString(" " + l.logicalList[index-1] + " ") - } - b.WriteString(where.String()) - } - return b.String() -} - -// Check 判断条件里是包含tag的值，例如：tagName: bk_biz_id，tagValue：[1, 2]，bk_biz_id = 1 和 bk_biz_id = 2 都符合 -func (l *WhereList) Check(tagName string, tagValue []string) bool { - tagMap := make(map[string]interface{}) - for _, v := range tagValue { - tagMap[v] = nil - } - for _, w := range l.whereList { - if w.Name == tagName && w.ValueType == StringType1 && w.Operator == EqualOperator { - if _, ok := tagMap[w.Value]; ok { - return true - } - } - } - return false -} - -// Where -type Where struct { - Name string - Value string - Operator string - ValueType ValueType -} - -// String -func (w *Where) String() string { - switch w.ValueType { - case NumType: - return fmt.Sprintf("%s %s %s", w.Name, w.Operator, w.Value) - case RegexpType: - // influxdb 中以 "/" 为分隔符，所以这里将正则中的 "/" 做个简单的转义 "\/" - return fmt.Sprintf("%s %s /%s/", w.Name, w.Operator, strings.ReplaceAll(w.Value, "/", "\\/")) - case TextType: - // 直接将长文本追加，作为一种特殊处理逻辑，influxQL 需要转义反斜杠 - return fmt.Sprintf("%s", strings.ReplaceAll(w.Value, "\\", "\\\\")) - default: - // 默认为字符串类型 - return fmt.Sprintf("%s %s '%s'", w.Name, w.Operator, w.Value) - } -} - -// 这里降低influxdb流量，主要不是根据时间减少点数，而是预先聚合减少series数量 -func generateSQL(field, measurement, db, aggregation string, whereList *WhereList, sLimit, limit int, dimensions []string, window time.Duration) (string, bool, bool) { - - var ( - groupingStr string - isWithGroupBy bool - withTag = ",*::tag" - limitStr string - sLimitStr string - aggField string - rpName string - isCountGroup bool - newAggregation string - ) - - newAggregation = aggregation - //rpName, field, newAggregation = GetRp(db, measurement, field, aggregation, window, whereList) - - // 根据RP重新生成measurement - if rpName != "" { - measurement = fmt.Sprintf("\"%s\".\"%s\"", rpName, measurement) - } else { - measurement = fmt.Sprintf("\"%s\"", measurement) - } - - // 存在聚合条件，需要增加聚合 - if newAggregation != "" && window != 0 { - var groupList []string - isWithGroupBy = true - isCountGroup = aggregation == COUNT - - if len(dimensions) > 0 { - for _, d := range dimensions { - groupList = append(groupList, fmt.Sprintf("\"%s\"", d)) - } - } - groupList = append(groupList, "time("+window.String()+")") - groupingStr = " group by " + strings.Join(groupList, ",") - withTag = "" - aggField = fmt.Sprintf("%s(\"%s\")", newAggregation, field) - // 由于此处存在聚合，所以可以直接使用influxdb的limit能力 - if sLimit > 0 { - sLimitStr = fmt.Sprintf(" slimit %d", sLimit) - } - } else { - aggField = fmt.Sprintf("\"%s\"", field) - } - - if limit > 0 { - limitStr = fmt.Sprintf(" limit %d", limit) - } - - return fmt.Sprintf("select %s as %s,time as %s%s from %s %s%s%s%s", - aggField, ResultColumnName, TimeColumnName, withTag, measurement, whereList.String(), groupingStr, limitStr, sLimitStr, - ), isWithGroupBy, isCountGroup -} - -// do -func do(ctx context.Context) []string { - metric, err := MetricMappingFromContext(ctx, ReferenceName) - if err != nil { - panic(err) - } - dataIDs := NewDataIDFilter(metric).FilterByBizIDs(2).FilterByProjectIDs(). - FilterByClusterIDs().Values() - - tableList := make([]*consul.TableID, 0) - for _, dataID := range dataIDs { - tableInfo := GetTableIDsByDataID(dataID) - if len(tableInfo) == 0 { - continue - } - tableList = append(tableList, tableInfo...) - } - - var res []string - where := NewWhereList() - for _, t := range tableList { - sql, _, _ := generateSQL("value", metric, t.DB, "max", where, 100, 100, nil, 5*time.Minute) - res = append(res, sql) - } - return res -} - -// TestGetTableIDByDBAndMeasurement -func TestGetTableIDByDBAndMeasurement(t *testing.T) { - stubs := FakeConsulRouter(t) - defer stubs.Reset() - - metricName := []string{"node_memory_Cached_bytes", "node_filesystem_files"} - - queryInfo := QueryInfo{ - DB: "", - Measurement: "", - DataIDList: nil, - IsPivotTable: false, - } - - var wg sync.WaitGroup - var wgCh sync.WaitGroup - var num = 1000 - - var checkSQL = map[string]string{ - "node_filesystem_files": "select max(\"value\") as _value,time as _time from \"node_filesystem_files\" group by time(5m0s) limit 100 slimit 100", - "node_memory_Cached_bytes": "select max(\"value\") as _value,time as _time from \"node_memory_Cached_bytes\" group by time(5m0s) limit 100 slimit 100", - } - var ch = map[string]chan []string{ - "node_filesystem_files": make(chan []string, 0), - "node_memory_Cached_bytes": make(chan []string, 0), - } - - wg.Add(1) - go func() { - defer wg.Done() - num = rand.Intn(50) - for i := 0; i < num; i++ { - go func() { - time.Sleep(time.Duration(num) * time.Millisecond) - data, err := consul.ReloadRouterInfo() - assert.Nil(t, err) - ReloadTableInfos(data) - - metricData, err := consul.ReloadMetricInfo() - assert.Nil(t, err) - ReloadMetricRouter(metricData) - }() - } - fmt.Printf("fresh %d\n", num) - time.Sleep(time.Second) - }() - - for _, metric := range metricName { - wgCh.Add(1) - go func(metric string) { - defer wgCh.Done() - for c := range ch[metric] { - for _, sql := range c { - _, err := influxql.ParseQuery(sql) - assert.Equal(t, sql, checkSQL[metric]) - assert.Nil(t, err) - t.Log(sql) - } - } - }(metric) - - ctx := context.Background() - ctx = QueryInfoIntoContext(ctx, ReferenceName, &queryInfo) - ctx = MetricMappingIntoContext(ctx, ReferenceName, metric) - - var ticker = time.NewTicker(5 * time.Second) - go func(metric string) { - defer ticker.Stop() - for { - for i := 0; i < num; i++ { - wg.Add(1) - go func(metric string) { - defer wg.Done() - ch[metric] <- do(ctx) - }(metric) - } - select { - case <-ticker.C: - close(ch[metric]) - return - } - } - }(metric) - } - wg.Wait() - fmt.Println(ch) - wgCh.Wait() -} diff --git a/pkg/unify-query/kvstore/bbolt/bbolt.go b/pkg/unify-query/kvstore/bbolt/bbolt.go index a1029e6d4..c41ce39b8 100644 --- a/pkg/unify-query/kvstore/bbolt/bbolt.go +++ b/pkg/unify-query/kvstore/bbolt/bbolt.go @@ -101,6 +101,10 @@ func (c *Client) Open() error { return fmt.Errorf("unable to open boltdb: %w", err) } + if db == nil { + return fmt.Errorf("open boltdb is empty") + } + // New Bucket err = db.Update(func(tx *bolt.Tx) error { _, bucketErr := tx.CreateBucketIfNotExists(c.BucketName) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 401352025..69783771d 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -339,10 +339,10 @@ func (i *Instance) Table(query *metadata.Query) string { } // QueryRawData 直接查询原始返回 -func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, err error) { +func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, start, end time.Time, dataCh chan<- map[string]any) (total int64, resultTableOptions metadata.ResultTableOptions, err error) { defer func() { if r := recover(); r != nil { - err = fmt.Errorf("doris query error: %s", r) + err = fmt.Errorf("doris query panic: %s", r) } }() @@ -367,6 +367,15 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } } + if len(query.ResultTableOptions) > 0 { + option := query.ResultTableOptions.GetOption(query.TableID, "") + if option != nil { + if option.From != nil { + query.From = *option.From + } + } + } + queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) if err != nil { return @@ -397,8 +406,10 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star list[KeyTableID] = query.TableID list[KeyDataLabel] = query.DataLabel - if query.HighLight.Enable { - list[KeyHighLight] = "" + if query.HighLight != nil { + if query.HighLight.Enable { + list[KeyHighLight] = "" + } } dataCh <- list diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 9add38bf0..a676a0321 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -26,17 +26,15 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) -func TestInstance_QueryRaw(t *testing.T) { +func TestInstance_QuerySeriesSet(t *testing.T) { ctx := metadata.InitHashID(context.Background()) ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", - "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) end := time.UnixMilli(1730118889181) @@ -51,28 +49,7 @@ func TestInstance_QueryRaw(t *testing.T) { query *metadata.Query expected string }{ - "query with in": { - query: &metadata.Query{ - DataSource: datasource, - TableID: tableID, - DB: db, - DataLabel: db, - MetricName: field, - BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", - OffsetInfo: metadata.OffSetInfo{Limit: 10}, - AllConditions: metadata.AllConditions{ - { - { - DimensionName: "namespace", - Operator: metadata.ConditionEqual, - Value: []string{"gz100", "bgp2-new"}, - }, - }, - }, - }, - expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:32:00","_startTime_":"2024-10-28 20:31:00","_timestamp_":1730118660000,"_value_":269,"dtEventTime":"2024-10-28 20:31:00","dtEventTimeStamp":1730118660000,"localTime":"2024-10-28 20:32:03","login_rate":269,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:29:00","_startTime_":"2024-10-28 20:28:00","_timestamp_":1730118480000,"_value_":271,"dtEventTime":"2024-10-28 20:28:00","dtEventTimeStamp":1730118480000,"localTime":"2024-10-28 20:29:03","login_rate":271,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:30:00","_startTime_":"2024-10-28 20:29:00","_timestamp_":1730118540000,"_value_":267,"dtEventTime":"2024-10-28 20:29:00","dtEventTimeStamp":1730118540000,"localTime":"2024-10-28 20:30:02","login_rate":267,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:31:00","_startTime_":"2024-10-28 20:30:00","_timestamp_":1730118600000,"_value_":274,"dtEventTime":"2024-10-28 20:30:00","dtEventTimeStamp":1730118600000,"localTime":"2024-10-28 20:31:04","login_rate":274,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:28:00","_startTime_":"2024-10-28 20:27:00","_timestamp_":1730118420000,"_value_":279,"dtEventTime":"2024-10-28 20:27:00","dtEventTimeStamp":1730118420000,"localTime":"2024-10-28 20:28:03","login_rate":279,"namespace":"gz100","thedate":20241028}]`, - }, - "count by namespace": { + "count by namespace with mysql": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, @@ -86,9 +63,9 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"bgp2"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"cq100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"gz100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn0-new"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn1"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"hn10"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"nj100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"njloadtest"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"pbe"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"tj100"},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_value_":5,"namespace":"tj101"}]`, + expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"bgp2"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"cq100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"gz100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn0-new"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn1"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"hn10"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"nj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"njloadtest"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"pbe"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj100"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null},{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"},{"name":"namespace","value":"tj101"}],"samples":[{"value":5,"timestamp":1730118589181}],"exemplars":null,"histograms":null}]`, }, - "count with 1m": { + "count with 1m with mysql": { query: &metadata.Query{ DataSource: datasource, TableID: tableID, @@ -102,9 +79,77 @@ func TestInstance_QueryRaw(t *testing.T) { }, }, }, - expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118600000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118660000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118720000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118780000,"_value_":11},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_timestamp_":1730118840000,"_value_":11}]`, + expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"}],"samples":[{"value":11,"timestamp":1730118600000},{"value":11,"timestamp":1730118660000},{"value":11,"timestamp":1730118720000},{"value":11,"timestamp":1730118780000},{"value":11,"timestamp":1730118840000}],"exemplars":null,"histograms":null}]`, }, - "count by doris": { + } { + t.Run(name, func(t *testing.T) { + ctx = metadata.InitHashID(ctx) + if c.query.DB == "" { + c.query.DB = db + } + if c.query.Field == "" { + c.query.Field = field + } + + set := ins.QuerySeriesSet(ctx, c.query, start, end) + ts, err := mock.SeriesSetToTimeSeries(set) + assert.Nil(t, err) + + actual, err := json.Marshal(ts) + assert.Nil(t, err) + + fmt.Println(string(actual)) + + assert.JSONEq(t, c.expected, string(actual)) + }) + } +} + +func TestInstance_QueryRaw(t *testing.T) { + + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) + + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + }) + + end := time.UnixMilli(1730118889181) + start := time.UnixMilli(1730118589181) + + datasource := "bkdata" + db := "132_lol_new_login_queue_login_1min" + field := "login_rate" + tableID := db + ".default" + + for name, c := range map[string]struct { + query *metadata.Query + expected string + }{ + "query with in": { + query: &metadata.Query{ + DataSource: datasource, + TableID: tableID, + DB: db, + DataLabel: db, + MetricName: field, + BkSqlCondition: "`namespace` IN ('gz100', 'bgp2\\-new')", + OffsetInfo: metadata.OffSetInfo{Limit: 10}, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "namespace", + Operator: metadata.ConditionEqual, + Value: []string{"gz100", "bgp2-new"}, + }, + }, + }, + }, + expected: `[{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:32:00","_startTime_":"2024-10-28 20:31:00","_timestamp_":1730118660000,"_value_":269,"dtEventTime":"2024-10-28 20:31:00","dtEventTimeStamp":1730118660000,"localTime":"2024-10-28 20:32:03","login_rate":269,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:29:00","_startTime_":"2024-10-28 20:28:00","_timestamp_":1730118480000,"_value_":271,"dtEventTime":"2024-10-28 20:28:00","dtEventTimeStamp":1730118480000,"localTime":"2024-10-28 20:29:03","login_rate":271,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:30:00","_startTime_":"2024-10-28 20:29:00","_timestamp_":1730118540000,"_value_":267,"dtEventTime":"2024-10-28 20:29:00","dtEventTimeStamp":1730118540000,"localTime":"2024-10-28 20:30:02","login_rate":267,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:31:00","_startTime_":"2024-10-28 20:30:00","_timestamp_":1730118600000,"_value_":274,"dtEventTime":"2024-10-28 20:30:00","dtEventTimeStamp":1730118600000,"localTime":"2024-10-28 20:31:04","login_rate":274,"namespace":"gz100","thedate":20241028},{"__data_label":"132_lol_new_login_queue_login_1min","__index":"132_lol_new_login_queue_login_1min","__result_table":"132_lol_new_login_queue_login_1min.default","_endTime_":"2024-10-28 20:28:00","_startTime_":"2024-10-28 20:27:00","_timestamp_":1730118420000,"_value_":279,"dtEventTime":"2024-10-28 20:27:00","dtEventTimeStamp":1730118420000,"localTime":"2024-10-28 20:28:03","login_rate":279,"namespace":"gz100","thedate":20241028}]`, + }, + "query raw by doris": { query: &metadata.Query{ TableID: "5000140_bklog_container_log_demo_analysis.doris", DB: "5000140_bklog_container_log_demo_analysis", @@ -133,7 +178,7 @@ func TestInstance_QueryRaw(t *testing.T) { close(dataCh) }() - _, err := ins.QueryRawData(ctx, c.query, start, end, dataCh) + _, _, err := ins.QueryRawData(ctx, c.query, start, end, dataCh) assert.Nil(t, err) }() @@ -331,27 +376,28 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", }, - { - name: "aggregate multi function", - query: &metadata.Query{ - DB: "132_hander_opmon_avg", - Field: "value", - Aggregates: metadata.Aggregates{ - { - Name: "sum", - Field: "value", - Window: time.Second * 15, - }, - { - Name: "count", - Field: "other", - Window: time.Hour, - }, - }, - }, - - expected: "", - }, + //{ + // name: "aggregate multi function", + // query: &metadata.Query{ + // DB: "132_hander_opmon_avg", + // Field: "value", + // Aggregates: metadata.Aggregates{ + // { + // Name: "sum", + // Field: "value", + // Window: time.Second * 15, + // }, + // { + // Name: "count", + // Field: "other", + // Window: time.Hour, + // }, + // }, + // }, + // + // // TODO 适配多字段查询特性 + // expected: "", + //}, { name: "query raw", query: &metadata.Query{ diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index 1311f3348..a10a72b7c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -77,7 +77,7 @@ func TestQsToDsl(t *testing.T) { } return "" }) - query, err := qs.Parser() + query, err := qs.ToDSL() if err == nil { if query != nil { body, _ := query.Source() From 1e9a09879082450810cd2a22d83745d0f0df0a64 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 14:48:07 +0800 Subject: [PATCH 065/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/handler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index 5ead655e3..1d0b44a6e 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -294,6 +294,11 @@ func HandlerQueryTs(c *gin.Context) { return } + // metadata 中的 spaceUid 是从 header 头信息中获取，header 如果有的话，覆盖参数里的 + if user.SpaceUid != "" { + query.SpaceUid = user.SpaceUid + } + queryStr, _ := json.Marshal(query) span.Set("query-body", string(queryStr)) span.Set("query-body-size", len(queryStr)) From 46dbe4cde419ee743af54b1ea3fc32d57d6efbd3 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 15:12:50 +0800 Subject: [PATCH 066/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance.go | 2 ++ pkg/unify-query/tsdb/bksql/instance_test.go | 8 ++++---- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 69783771d..19397a4cd 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -453,6 +453,8 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st return storage.ErrSeriesSet(err) } + span.Set("query-sql", sql) + data, err := i.sqlQuery(ctx, sql, span) if err != nil { return storage.ErrSeriesSet(err) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index a676a0321..8ef313f9b 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -32,7 +32,7 @@ func TestInstance_QuerySeriesSet(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ - "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) @@ -111,7 +111,7 @@ func TestInstance_QueryRaw(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ - "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) @@ -850,7 +850,7 @@ func TestInstance_QueryLabelNames_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ - "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, }) @@ -902,7 +902,7 @@ func TestInstance_QueryLabelValues_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ - "SHOW CREATE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, }) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 540ece00f..fb198a09c 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -65,7 +65,7 @@ func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { } func (d *DorisSQLExpr) DescribeTable(table string) string { - return fmt.Sprintf("SHOW CREATE %s", table) + return fmt.Sprintf("SHOW CREATE TABLE %s", table) } // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 From c37ff6da2868088f2931c71f309c14055d2883b7 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 15:55:38 +0800 Subject: [PATCH 067/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 8 +++- pkg/unify-query/tsdb/bksql/instance.go | 44 ++++++++++++--------- pkg/unify-query/tsdb/bksql/instance_test.go | 43 ++++++++++++++++++++ pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 18 ++++++--- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 6 ++- 5 files changed, 92 insertions(+), 27 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 66b7716dc..1721102e6 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -107,8 +107,12 @@ func (f *QueryFactory) Table() string { return table } -func (f *QueryFactory) DescribeTable() string { - return f.expr.DescribeTable(f.Table()) +func (f *QueryFactory) DescribeTableSQL() string { + return f.expr.DescribeTableSQL(f.Table()) +} + +func (f *QueryFactory) FieldMap() map[string]string { + return f.expr.FieldMap() } func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 19397a4cd..2162beacd 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -11,6 +11,7 @@ package bksql import ( "context" + "encoding/json" "fmt" "strings" "time" @@ -111,14 +112,18 @@ func (i *Instance) checkResult(res *Result) error { return nil } -func (i *Instance) sqlQuery(ctx context.Context, sql string, span *trace.Span) (*QuerySyncResultData, error) { +func (i *Instance) sqlQuery(ctx context.Context, sql string) (*QuerySyncResultData, error) { var ( data *QuerySyncResultData - ok bool - err error + ok bool + err error + span *trace.Span ) + ctx, span = trace.NewSpan(ctx, "sql-query") + defer span.End(&err) + if sql == "" { return data, nil } @@ -278,14 +283,14 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad return res, nil } -func (i *Instance) getFieldsMap(ctx context.Context, sql string, span *trace.Span) (map[string]string, error) { +func (i *Instance) getFieldsMap(ctx context.Context, sql string) (map[string]string, error) { fieldsMap := make(map[string]string) if sql == "" { return nil, nil } - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return nil, err } @@ -311,18 +316,23 @@ func (i *Instance) getFieldsMap(ctx context.Context, sql string, span *trace.Spa return fieldsMap, nil } -func (i *Instance) InitQueryFactory(ctx context.Context, query *metadata.Query, start, end time.Time, span *trace.Span) (*QueryFactory, error) { +func (i *Instance) InitQueryFactory(ctx context.Context, query *metadata.Query, start, end time.Time) (*QueryFactory, error) { + var err error + + ctx, span := trace.NewSpan(ctx, "instance-init-query-factory") + defer span.End(&err) f := NewQueryFactory(ctx, query).WithRangeTime(start, end) // 只有 Doris 才需要获取字段表结构 if query.Measurement == sqlExpr.Doris { - fieldsMap, err := i.getFieldsMap(ctx, f.DescribeTable(), span) + fieldsMap, err := i.getFieldsMap(ctx, f.DescribeTableSQL()) if err != nil { return f, err } - span.Set("table_fields_map", fieldsMap) + out, _ := json.Marshal(fieldsMap) + span.Set("table_fields_map", string(out)) f.WithFieldsMap(fieldsMap) } @@ -376,7 +386,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } } - queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) if err != nil { return } @@ -385,7 +395,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return } @@ -444,7 +454,7 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st } } - queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) if err != nil { return storage.ErrSeriesSet(err) } @@ -453,9 +463,7 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st return storage.ErrSeriesSet(err) } - span.Set("query-sql", sql) - - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return storage.ErrSeriesSet(err) } @@ -505,7 +513,7 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s // 取字段名不需要返回数据，但是 size 不能使用 0，所以还是用 1 query.Size = 1 - queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) if err != nil { return nil, err } @@ -515,7 +523,7 @@ func (i *Instance) QueryLabelNames(ctx context.Context, query *metadata.Query, s return nil, err } - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return nil, err } @@ -546,7 +554,7 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, }, } - queryFactory, err := i.InitQueryFactory(ctx, query, start, end, span) + queryFactory, err := i.InitQueryFactory(ctx, query, start, end) if err != nil { return nil, err } @@ -555,7 +563,7 @@ func (i *Instance) QueryLabelValues(ctx context.Context, query *metadata.Query, return nil, err } - data, err := i.sqlQuery(ctx, sql, span) + data, err := i.sqlQuery(ctx, sql) if err != nil { return nil, err } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 8ef313f9b..8abd563aa 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -22,10 +22,53 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) +func TestInstance_ShowCreateTable(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + ins := createTestInstance(ctx) + + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `132_lol_new_login_queue_login_1min`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":20,"external_api_call_time_mills":{"bkbase_auth_api":64,"bkbase_meta_api":6,"bkbase_apigw_api":25},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c1","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c2","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":27,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":56,"match_query_routing_rule":0,"check_permission":66,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":13168,"timetaken":0.161,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + }) + + end := time.UnixMilli(1730118889181) + start := time.UnixMilli(1730118589181) + + datasource := "bklog" + db := "132_lol_new_login_queue_login_1min" + measurement := "doris" + + for name, c := range map[string]struct { + query *metadata.Query + expected string + }{ + "field map": { + query: &metadata.Query{ + DataSource: datasource, + DB: db, + Measurement: measurement, + }, + expected: `{"__ext":"varchar(65533)","__shard_key__":"bigint","c1":"text","c2":"text","cloudid":"double","dteventtime":"varchar(32)","dteventtimestamp":"bigint","file":"varchar(65533)","gseindex":"double","iterationindex":"double","level":"varchar(65533)","localtime":"varchar(32)","log":"text","message":"varchar(65533)","path":"text","report_time":"varchar(65533)","serverip":"varchar(65533)","thedate":"int","time":"text","trace_id":"varchar(65533)"}`, + }, + } { + t.Run(name, func(t *testing.T) { + ctx = metadata.InitHashID(ctx) + + ctx, span := trace.NewSpan(ctx, "test") + fact, err := ins.InitQueryFactory(ctx, c.query, start, end, span) + assert.Nil(t, err) + + actual, _ := json.Marshal(fact.FieldMap()) + assert.JSONEq(t, c.expected, string(actual)) + + }) + } +} + func TestInstance_QuerySeriesSet(t *testing.T) { ctx := metadata.InitHashID(context.Background()) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 1477f94aa..6843e94e6 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -49,8 +49,10 @@ type SQLExpr interface { ParserAllConditions(allConditions metadata.AllConditions) (string, error) // ParserAggregatesAndOrders 解析聚合条件生成SQL条件表达式 ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, error) - // DescribeTable 返回当前表结构 - DescribeTable(table string) string + // DescribeTableSQL 返回当前表结构 + DescribeTableSQL(table string) string + // FieldMap 返回当前表结构 + FieldMap() map[string]string } // SQL表达式注册管理相关变量 @@ -106,12 +108,16 @@ func UnRegister(key string) { type DefaultSQLExpr struct { encodeFunc func(string) string - fieldsMap map[string]string + fieldMap map[string]string timeField string valueField string } +func (d *DefaultSQLExpr) FieldMap() map[string]string { + return d.fieldMap +} + func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -123,8 +129,8 @@ func (d *DefaultSQLExpr) WithEncode(fn func(string) string) SQLExpr { return d } -func (d *DefaultSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { - d.fieldsMap = fieldsMap +func (d *DefaultSQLExpr) WithFieldsMap(fieldMap map[string]string) SQLExpr { + d.fieldMap = fieldMap return d } @@ -253,7 +259,7 @@ func (d *DefaultSQLExpr) ParserAllConditions(allConditions metadata.AllCondition return "", nil } -func (d *DefaultSQLExpr) DescribeTable(table string) string { +func (d *DefaultSQLExpr) DescribeTableSQL(table string) string { return "" } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index fb198a09c..a84ccc05f 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -36,6 +36,10 @@ type DorisSQLExpr struct { var _ SQLExpr = (*DorisSQLExpr)(nil) +func (d *DorisSQLExpr) FieldMap() map[string]string { + return d.fieldsMap +} + func (d *DorisSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -64,7 +68,7 @@ func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { return d.walk(expr) } -func (d *DorisSQLExpr) DescribeTable(table string) string { +func (d *DorisSQLExpr) DescribeTableSQL(table string) string { return fmt.Sprintf("SHOW CREATE TABLE %s", table) } From e909db22976904d7218e49a23a062dae591b0dae Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 17:10:45 +0800 Subject: [PATCH 068/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/querystring/parser.go | 2 +- pkg/unify-query/tsdb/bksql/format.go | 18 +++++++++++++++++- pkg/unify-query/tsdb/bksql/instance.go | 1 + pkg/unify-query/tsdb/bksql/instance_test.go | 4 +--- .../tsdb/elasticsearch/query_string.go | 2 +- .../tsdb/elasticsearch/query_string_test.go | 17 ++++++++++++----- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pkg/unify-query/internal/querystring/parser.go b/pkg/unify-query/internal/querystring/parser.go index 998ae80a1..f2943edf8 100644 --- a/pkg/unify-query/internal/querystring/parser.go +++ b/pkg/unify-query/internal/querystring/parser.go @@ -18,7 +18,7 @@ import ( // Parse querystring and return Expr func Parse(query string) (Expr, error) { - if query == "" { + if query == "" || query == "*" { return nil, nil } diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 1721102e6..b11ee9212 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -21,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) @@ -153,7 +154,7 @@ func (f *QueryFactory) BuildWhere() (string, error) { } // QueryString to sql - if f.query.QueryString != "" { + if f.query.QueryString != "" && f.query.QueryString != "*" { qs, err := f.expr.ParserQueryString(f.query.QueryString) if err != nil { return "", err @@ -180,13 +181,27 @@ func (f *QueryFactory) BuildWhere() (string, error) { } func (f *QueryFactory) SQL() (sql string, err error) { + var ( + span *trace.Span + ) + + _, span = trace.NewSpan(f.ctx, "make-sql") + defer span.End(&err) + selectFields, groupFields, orderFields, err := f.expr.ParserAggregatesAndOrders(f.query.Aggregates, f.orders) if err != nil { return } + span.Set("select-fields", selectFields) + span.Set("group-fields", groupFields) + span.Set("order-fields", orderFields) + sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), f.Table()) whereString, err := f.BuildWhere() + + span.Set("where-string", whereString) + if err != nil { return } @@ -208,6 +223,7 @@ func (f *QueryFactory) SQL() (sql string, err error) { sql += fmt.Sprintf(" LIMIT %d", f.query.Size) } + span.Set("sql", sql) return } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 2162beacd..972e01e7d 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -147,6 +147,7 @@ func (i *Instance) sqlQuery(ctx context.Context, sql string) (*QuerySyncResultDa return data, fmt.Errorf("queryAsyncResult type is error: %T", res.Data) } + span.Set("result-size", len(data.List)) return data, nil } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 8abd563aa..c79ab8b31 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -22,7 +22,6 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" ) @@ -58,8 +57,7 @@ func TestInstance_ShowCreateTable(t *testing.T) { t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) - ctx, span := trace.NewSpan(ctx, "test") - fact, err := ins.InitQueryFactory(ctx, c.query, start, end, span) + fact, err := ins.InitQueryFactory(ctx, c.query, start, end) assert.Nil(t, err) actual, _ := json.Marshal(fact.FieldMap()) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index f39b424aa..450805bef 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -53,7 +53,7 @@ func (s *QueryString) ToDSL() (elastic.Query, error) { } }() - if s.q == "" || s.q == "*" { + if s.q == "" { return nil, nil } diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go index a10a72b7c..9d9b59612 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string_test.go @@ -59,7 +59,8 @@ func TestQsToDsl(t *testing.T) { expected: `{"query_string":{"analyze_wildcard":true,"fields":["*", "__*"],"lenient":true,"query":"sync_spaces AND -keyword AND -BKLOGAPI"}}`, }, { - q: `*`, + q: `*`, + expected: `{"query_string":{"fields":["*","__*"],"analyze_wildcard":true,"lenient":true,"query":"*"}}`, }, { q: ``, @@ -80,10 +81,16 @@ func TestQsToDsl(t *testing.T) { query, err := qs.ToDSL() if err == nil { if query != nil { - body, _ := query.Source() - bodyJson, _ := json.Marshal(body) - bodyString := string(bodyJson) - assert.JSONEq(t, c.expected, bodyString) + body, err := query.Source() + assert.Nil(t, err) + + if body != nil { + bodyJson, _ := json.Marshal(body) + bodyString := string(bodyJson) + assert.JSONEq(t, c.expected, bodyString) + } else { + assert.Empty(t, c.expected) + } } } else { assert.Equal(t, c.err, err) From 976c536877a45e83f3b7313fbefe8c087a19b643 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 17:11:35 +0800 Subject: [PATCH 069/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index a84ccc05f..dbda7e8d4 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -338,10 +338,6 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { right string ) - if d == nil { - return "", nil - } - switch c := e.(type) { case *querystring.NotExpr: left, err = d.walk(c.Expr) From 0698752a9d48d954a38c880fc6d5ec8120aa5184 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 10 Apr 2025 18:18:20 +0800 Subject: [PATCH 070/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 5 +++++ pkg/unify-query/tsdb/bksql/instance.go | 10 ++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 18 +++++++++++----- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 23 +++++++++++++++------ 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index b11ee9212..319ca5e11 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -100,6 +100,11 @@ func (f *QueryFactory) WithFieldsMap(m map[string]string) *QueryFactory { return f } +func (f *QueryFactory) WithKeepColumns(cols []string) *QueryFactory { + f.expr.WithKeepColumns(cols) + return f +} + func (f *QueryFactory) Table() string { table := fmt.Sprintf("`%s`", f.query.DB) if f.query.Measurement != "" { diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 972e01e7d..f1c6d7b96 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -332,10 +332,18 @@ func (i *Instance) InitQueryFactory(ctx context.Context, query *metadata.Query, return f, err } + // 只能使用在表结构的字段才能使用 + var keepColumns []string + for _, k := range query.Source { + if _, ok := fieldsMap[k]; ok { + keepColumns = append(keepColumns, k) + } + } out, _ := json.Marshal(fieldsMap) span.Set("table_fields_map", string(out)) - f.WithFieldsMap(fieldsMap) + span.Set("keep-columns", keepColumns) + f.WithFieldsMap(fieldsMap).WithKeepColumns(keepColumns) } return f, nil diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 6843e94e6..9d7b24773 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -37,6 +37,8 @@ var ( // SQLExpr 定义SQL表达式生成接口 // 接口包含字段映射、查询字符串解析、全条件解析等功能 type SQLExpr interface { + // WithKeepColumns 设置保留字段 + WithKeepColumns([]string) SQLExpr // WithFieldsMap 设置字段类型 WithFieldsMap(fieldsMap map[string]string) SQLExpr // WithEncode 字段转换方法 @@ -108,16 +110,13 @@ func UnRegister(key string) { type DefaultSQLExpr struct { encodeFunc func(string) string - fieldMap map[string]string + keepColumns []string + fieldMap map[string]string timeField string valueField string } -func (d *DefaultSQLExpr) FieldMap() map[string]string { - return d.fieldMap -} - func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -134,6 +133,15 @@ func (d *DefaultSQLExpr) WithFieldsMap(fieldMap map[string]string) SQLExpr { return d } +func (d *DefaultSQLExpr) WithKeepColumns(cols []string) SQLExpr { + d.keepColumns = cols + return d +} + +func (d *DefaultSQLExpr) FieldMap() map[string]string { + return d.fieldMap +} + // ParserQueryString 解析查询字符串（当前实现返回空） func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { return "", nil diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index dbda7e8d4..4d581c75b 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -31,15 +31,12 @@ type DorisSQLExpr struct { timeField string valueField string - fieldsMap map[string]string + keepColumns []string + fieldsMap map[string]string } var _ SQLExpr = (*DorisSQLExpr)(nil) -func (d *DorisSQLExpr) FieldMap() map[string]string { - return d.fieldsMap -} - func (d *DorisSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -56,6 +53,15 @@ func (d *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { return d } +func (d *DorisSQLExpr) WithKeepColumns(cols []string) SQLExpr { + d.keepColumns = cols + return d +} + +func (d *DorisSQLExpr) FieldMap() map[string]string { + return d.fieldsMap +} + func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { expr, err := querystring.Parse(qs) if err != nil { @@ -131,7 +137,12 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, } if len(selectFields) == 0 { - selectFields = append(selectFields, SelectAll) + if len(d.keepColumns) > 0 { + selectFields = append(selectFields, d.keepColumns...) + } else { + selectFields = append(selectFields, SelectAll) + } + if valueField != "" { selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", valueField, Value)) } From 1dafbbcff9dc445141b4df1b8f67d1b8da5f9370 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 11 Apr 2025 11:18:40 +0800 Subject: [PATCH 071/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance.go | 6 ----- pkg/unify-query/tsdb/bksql/instance_test.go | 15 +++++++++++++ pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 8 ++++++- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 25 ++++++++++++++++++--- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index f1c6d7b96..6f2e712b1 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -425,12 +425,6 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star list[KeyTableID] = query.TableID list[KeyDataLabel] = query.DataLabel - if query.HighLight != nil { - if query.HighLight.Enable { - list[KeyHighLight] = "" - } - } - dataCh <- list } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index c79ab8b31..a18a36730 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -417,6 +417,21 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", }, + { + name: "aggregate cardinality", + query: &metadata.Query{ + DB: "2_bklog_bkunify_query_doris", + Measurement: "", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "cardinality", + }, + }, + }, + + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, //{ // name: "aggregate multi function", // query: &metadata.Query{ diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 9d7b24773..8026b5083 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -167,7 +167,13 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate if valueField == "" { valueField = SelectAll } - selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + + switch agg.Name { + case "cardinality": + selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + default: + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + } if agg.Window > 0 { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 4d581c75b..3e64cf5a2 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -78,11 +78,25 @@ func (d *DorisSQLExpr) DescribeTableSQL(table string) string { return fmt.Sprintf("SHOW CREATE TABLE %s", table) } +func (d *DorisSQLExpr) aggregateTransform(aggregates metadata.Aggregates) metadata.Aggregates { + newAggregates := make(metadata.Aggregates, 0) + for _, agg := range aggregates { + switch agg.Name { + case "cardinality": + default: + newAggregates = append(newAggregates, agg) + } + } + return newAggregates +} + // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { valueField, _ := d.dimTransform(d.valueField) - for _, agg := range aggregates { + newAggregates := d.aggregateTransform(aggregates) + + for _, agg := range newAggregates { for _, dim := range agg.Dimensions { var ( isObject = false @@ -104,10 +118,15 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, if valueField == "" { valueField = SelectAll } - selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) - if agg.Window > 0 { + switch agg.Name { + case "cardinality": + selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + default: + selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) + } + if agg.Window > 0 { // 获取时区偏移量 var offsetMinutes int // 如果是按天聚合，则增加时区偏移量 From 4150e4cc9911dbf581f700b0bf103f6db6fafc90 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 11 Apr 2025 12:03:03 +0800 Subject: [PATCH 072/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance_test.go | 38 +++++++++++++++++++++ pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 10 ++++++ pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 11 ++++++ 3 files changed, 59 insertions(+) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index a18a36730..6e4af57a6 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -428,6 +428,20 @@ func TestInstance_bkSql(t *testing.T) { Name: "cardinality", }, }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, }, expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", @@ -454,6 +468,30 @@ func TestInstance_bkSql(t *testing.T) { // // TODO 适配多字段查询特性 // expected: "", //}, + { + name: "query raw order ", + query: &metadata.Query{ + DB: "100133_ieod_logsearch4_errorlog_p", + Measurement: "doris", + Field: "value", + Size: 5, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, + }, + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `dtEventTimeStamp` DESC, `gseIndex` DESC, `iterationIndex` DESC LIMIT 5", + }, { name: "query raw", query: &metadata.Query{ diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 8026b5083..041c8247e 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -154,12 +154,15 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate return } + dimensionMap := make(map[string]struct{}) for _, agg := range aggregates { for _, dim := range agg.Dimensions { dim, err = d.dimTransform(dim) if err != nil { return } + dimensionMap[dim] = struct{}{} + selectFields = append(selectFields, dim) groupByFields = append(groupByFields, dim) } @@ -209,6 +212,13 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate } for _, order := range orders { + // 如果是聚合操作的话，只能使用维度进行排序 + if len(aggregates) > 0 { + if _, ok := dimensionMap[order.Name]; !ok { + continue + } + } + var orderField string switch order.Name { case FieldValue: diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 3e64cf5a2..342ca1953 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -96,6 +96,8 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, newAggregates := d.aggregateTransform(aggregates) + dimensionMap := make(map[string]struct{}) + for _, agg := range newAggregates { for _, dim := range agg.Dimensions { var ( @@ -111,6 +113,8 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, selectAlias = newDim } + dimensionMap[dim] = struct{}{} + selectFields = append(selectFields, selectAlias) groupByFields = append(groupByFields, newDim) } @@ -171,6 +175,13 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, } for _, order := range orders { + // 如果是聚合操作的话，只能使用维度进行排序 + if len(aggregates) > 0 { + if _, ok := dimensionMap[order.Name]; !ok { + continue + } + } + var orderField string switch order.Name { case FieldValue: From 5fcd0d09aed913adb14b0264d2e035d588cf63c7 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 11 Apr 2025 14:29:41 +0800 Subject: [PATCH 073/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/method.go | 2 +- pkg/unify-query/tsdb/bksql/instance_test.go | 98 ++++++++++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 43 ++++---- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 74 +++++++------- .../tsdb/elasticsearch/instance_test.go | 3 - 5 files changed, 159 insertions(+), 61 deletions(-) diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index 468b8fd1f..611221c79 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -51,7 +51,6 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) Dimensions: aggr.Dimensions, Without: aggr.Without, Args: aggr.VArgsList, - TimeZone: timezone, } if aggr.Window != "" { @@ -61,6 +60,7 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) } agg.Window = time.Duration(window) + agg.TimeZone = timezone } aggs = append(aggs, agg) } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 6e4af57a6..cd6882c8d 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -418,7 +418,7 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", }, { - name: "aggregate cardinality", + name: "aggregate cardinality with mysql", query: &metadata.Query{ DB: "2_bklog_bkunify_query_doris", Measurement: "", @@ -446,6 +446,102 @@ func TestInstance_bkSql(t *testing.T) { expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", }, + { + name: "aggregate date_histogram with mysql", + query: &metadata.Query{ + DB: "2_bklog_bkunify_query_doris", + Measurement: "", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "count", + }, + { + Name: "date_histogram", + Window: time.Minute * 5, + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, + }, + + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 300000 * 300000 - 0) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY (dtEventTimeStamp + 0) / 300000 * 300000 - 0 ORDER BY `_timestamp_` ASC", + }, + { + name: "aggregate cardinality with doris", + query: &metadata.Query{ + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "cardinality", + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, + }, + + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + }, + { + name: "aggregate date_histogram with doris", + query: &metadata.Query{ + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "gseIndex", + Aggregates: metadata.Aggregates{ + { + Name: "count", + }, + { + Name: "date_histogram", + Window: time.Minute * 5, + }, + }, + Orders: metadata.Orders{ + { + Name: "dtEventTimeStamp", + Ast: false, + }, + { + Name: "gseIndex", + Ast: false, + }, + { + Name: "iterationIndex", + Ast: false, + }, + }, + }, + + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + //{ // name: "aggregate multi function", // query: &metadata.Query{ diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 041c8247e..c7366c8ea 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -154,7 +154,11 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate return } - dimensionMap := make(map[string]struct{}) + var ( + window time.Duration + timezone string + dimensionMap = make(map[string]struct{}) + ) for _, agg := range aggregates { for _, dim := range agg.Dimensions { dim, err = d.dimTransform(dim) @@ -174,31 +178,36 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate switch agg.Name { case "cardinality": selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + // date_histogram 不支持无需进行函数聚合 + case "date_histogram": default: selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) } if agg.Window > 0 { + window = agg.Window + timezone = agg.TimeZone + } + } - var offsetMillis int - if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { - // 时间聚合函数兼容时区 - loc, locErr := time.LoadLocation(agg.TimeZone) - if locErr != nil { - loc = time.UTC - } - // 获取时区偏移量 - _, offset := time.Now().In(loc).Zone() - offsetMillis = offset * 1e3 + if window > 0 { + var offsetMillis int + if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(timezone) + if locErr != nil { + loc = time.UTC } + // 获取时区偏移量 + _, offset := time.Now().In(loc).Zone() + offsetMillis = offset * 1e3 + } - timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, offsetMillis, agg.Window.Milliseconds(), agg.Window.Milliseconds(), offsetMillis) - - groupByFields = append(groupByFields, timeField) - selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) + timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, offsetMillis, window.Milliseconds(), window.Milliseconds(), offsetMillis) - orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) - } + groupByFields = append(groupByFields, timeField) + selectFields = append(selectFields, fmt.Sprintf("MAX(%s) AS `%s`", timeField, TimeStamp)) + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) } if len(selectFields) == 0 { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 342ca1953..77fd2b366 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -78,27 +78,16 @@ func (d *DorisSQLExpr) DescribeTableSQL(table string) string { return fmt.Sprintf("SHOW CREATE TABLE %s", table) } -func (d *DorisSQLExpr) aggregateTransform(aggregates metadata.Aggregates) metadata.Aggregates { - newAggregates := make(metadata.Aggregates, 0) - for _, agg := range aggregates { - switch agg.Name { - case "cardinality": - default: - newAggregates = append(newAggregates, agg) - } - } - return newAggregates -} - // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { valueField, _ := d.dimTransform(d.valueField) - newAggregates := d.aggregateTransform(aggregates) - - dimensionMap := make(map[string]struct{}) - - for _, agg := range newAggregates { + var ( + window time.Duration + timezone string + dimensionMap = make(map[string]struct{}) + ) + for _, agg := range aggregates { for _, dim := range agg.Dimensions { var ( isObject = false @@ -126,37 +115,44 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, switch agg.Name { case "cardinality": selectFields = append(selectFields, fmt.Sprintf("COUNT(DISTINCT %s) AS `%s`", valueField, Value)) + // date_histogram 不支持无需进行函数聚合 + case "date_histogram": default: selectFields = append(selectFields, fmt.Sprintf("%s(%s) AS `%s`", strings.ToUpper(agg.Name), valueField, Value)) } if agg.Window > 0 { - // 获取时区偏移量 - var offsetMinutes int - // 如果是按天聚合，则增加时区偏移量 - if agg.Window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { - // 时间聚合函数兼容时区 - loc, locErr := time.LoadLocation(agg.TimeZone) - if locErr != nil { - loc = time.UTC - } - _, offset := time.Now().In(loc).Zone() - offsetMinutes = offset / 60 - } + window = agg.Window + timezone = agg.TimeZone + } + } - // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 - var timeField string - if int64(agg.Window.Seconds())%60 == 0 { - windowMinutes := int(agg.Window.Minutes()) - timeField = fmt.Sprintf(`((CAST((%s / 1000 + %d) / %d AS INT) * %d - %d) * 60 * 1000)`, ShardKey, offsetMinutes, windowMinutes, windowMinutes, offsetMinutes) - } else { - timeField = fmt.Sprintf(`CAST(%s / %d AS INT) * %d `, d.timeField, agg.Window.Milliseconds(), agg.Window.Milliseconds()) + if window > 0 { + // 获取时区偏移量 + var offsetMinutes int + // 如果是按天聚合，则增加时区偏移量 + if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { + // 时间聚合函数兼容时区 + loc, locErr := time.LoadLocation(timezone) + if locErr != nil { + loc = time.UTC } + _, offset := time.Now().In(loc).Zone() + offsetMinutes = offset / 60 + } - selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", timeField, TimeStamp)) - groupByFields = append(groupByFields, TimeStamp) - orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) + // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 + var timeField string + if int64(window.Seconds())%60 == 0 { + windowMinutes := int(window.Minutes()) + timeField = fmt.Sprintf(`((CAST((%s / 1000 + %d) / %d AS INT) * %d - %d) * 60 * 1000)`, ShardKey, offsetMinutes, windowMinutes, windowMinutes, offsetMinutes) + } else { + timeField = fmt.Sprintf(`CAST(%s / %d AS INT) * %d `, d.timeField, window.Milliseconds(), window.Milliseconds()) } + + selectFields = append(selectFields, fmt.Sprintf("%s AS `%s`", timeField, TimeStamp)) + groupByFields = append(groupByFields, TimeStamp) + orderByFields = append(orderByFields, fmt.Sprintf("`%s` ASC", TimeStamp)) } if len(selectFields) == 0 { diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index 0e4e8a706..fd2c90d12 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -485,9 +485,6 @@ func TestInstance_queryRawData(t *testing.T) { // nested query + query string 测试 + highlight `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"fields":["*","__*"],"lenient":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, - - - // "nested aggregate + query 测试 `{"_source":{"includes":["group","user.first","user.last"]},"aggregations":{"user":{"aggregations":{"_value":{"value_count":{"field":"user.first"}}},"nested":{"path":"user"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":17,"relation":"eq"},"max_score":null,"hits":[]},"aggregations":{"user":{"doc_count":18,"_value":{"value":18}}}}`, From 6f73ae7bc63e64d46d490d5ba90bb154f4dd2f59 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 11 Apr 2025 17:27:55 +0800 Subject: [PATCH 074/104] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=20tagvalues?= =?UTF-8?q?=20=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3=EF=BC=8C=E8=B6=85?= =?UTF-8?q?=E8=BF=87=E4=B8=80=E5=A4=A9=E7=9A=84=E4=BD=BF=E7=94=A8=20labelv?= =?UTF-8?q?alues=20=E6=9F=A5=E8=AF=A2=20#1010158081123432529?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/function/function.go | 14 ++++++ pkg/unify-query/mock/handler.go | 2 + pkg/unify-query/service/http/api.go | 50 +++++++++++++++---- pkg/unify-query/service/http/handler_test.go | 31 +++++++++++- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/pkg/unify-query/internal/function/function.go b/pkg/unify-query/internal/function/function.go index 13acd141f..f63d8a10e 100644 --- a/pkg/unify-query/internal/function/function.go +++ b/pkg/unify-query/internal/function/function.go @@ -13,6 +13,7 @@ import ( "fmt" "math" "strconv" + "strings" "time" "github.com/prometheus/prometheus/model/labels" @@ -25,6 +26,19 @@ const ( Nanosecond = "nanosecond" ) +func MatcherToString(matchers ...*labels.Matcher) string { + vals := make([]string, 0, len(matchers)) + for _, m := range matchers { + vals = append(vals, m.String()) + } + + if len(vals) == 0 { + return "" + } + + return fmt.Sprintf("{%s}", strings.Join(vals, ",")) +} + func MatcherToMetricName(matchers ...*labels.Matcher) string { for _, m := range matchers { if m.Name == labels.MetricName { diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index 3aac1eb30..6d9b51c07 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -224,6 +224,7 @@ func mockBkSQLHandler(ctx context.Context) { d, ok := BkSQL.Get(request.Sql) if !ok { err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) + log.Errorf(ctx, err.Error()) return } @@ -305,6 +306,7 @@ func mockVmHandler(ctx context.Context) { d, ok := Vm.Get(key) if !ok { err = fmt.Errorf(`vm mock data is empty in "%s"`, key) + log.Errorf(ctx, err.Error()) return } diff --git a/pkg/unify-query/service/http/api.go b/pkg/unify-query/service/http/api.go index f9e025a91..da70c1dab 100644 --- a/pkg/unify-query/service/http/api.go +++ b/pkg/unify-query/service/http/api.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/set" @@ -68,6 +69,10 @@ func HandlerFieldKeys(c *gin.Context) { span.Set("request-header", c.Request.Header) span.Set("request-data", paramsStr) + queryTs := infoParamsToQueryTs(ctx, params) + unit, startTime, endTime, err := function.QueryTimestamp(queryTs.Start, queryTs.End) + metadata.GetQueryParams(ctx).SetTime(startTime, endTime, unit) + queryRef, start, end, err := infoParamsToQueryRefAndTime(ctx, params) if err != nil { resp.failed(ctx, err) @@ -233,6 +238,10 @@ func HandlerTagValues(c *gin.Context) { lblMap sync.Map ) + left := end.Sub(start) + + span.Set("left", left) + for _, name := range params.Keys { lbl, _ := lblMap.LoadOrStore(name, set.New[string]()) queryRef.Range("", func(qry *metadata.Query) { @@ -242,16 +251,37 @@ func HandlerTagValues(c *gin.Context) { _ = p.Submit(func() { defer wg.Done() + instance := prometheus.GetTsDbInstance(ctx, qry) if instance == nil { return } - res, err := instance.QueryLabelValues(ctx, qry, name, start, end) + var ( + res []string + matcher *labels.Matcher + ) + + // 优化 vm 查询，超过 1d 使用直查接口 + if left >= 24*time.Hour && instance.InstanceType() == consul.VictoriaMetricsStorageType { + vmExpand := queryRef.ToVmExpand(ctx) + metadata.SetExpand(ctx, vmExpand) + + matcher, err = labels.NewMatcher(labels.MatchEqual, labels.MetricName, prometheus.ReferenceName) + + span.Set("direct-label-values-matcher", matcher.String()) + span.Set("direct-label-values-size", qry.Size) + span.Set("direct-label-values-size", vmExpand) + + res, err = instance.DirectLabelValues(ctx, name, start, end, qry.Size, matcher) + } else { + res, err = instance.QueryLabelValues(ctx, qry, name, start, end) + } if err != nil { return } + span.Set("result-size", len(res)) lbl.(*set.Set[string]).Add(res...) }) }) @@ -485,7 +515,7 @@ func HandlerLabelValues(c *gin.Context) { return } -func infoParamsToQueryRefAndTime(ctx context.Context, params *infos.Params) (queryRef metadata.QueryReference, start, end time.Time, err error) { +func infoParamsToQueryTs(ctx context.Context, params *infos.Params) *structured.QueryTs { var ( user = metadata.GetUser(ctx) ) @@ -509,16 +539,16 @@ func infoParamsToQueryRefAndTime(ctx context.Context, params *infos.Params) (que Timezone: params.Timezone, } - var unit string - unit, start, end, err = function.QueryTimestamp(params.Start, params.End) - if err != nil { - // 如果时间异常则使用最近 1h - end = time.Now() - start = end.Add(time.Hour * -1) - } + return queryTs +} +func infoParamsToQueryRefAndTime(ctx context.Context, params *infos.Params) (queryRef metadata.QueryReference, startTime, endTime time.Time, err error) { + var unit string + queryTs := infoParamsToQueryTs(ctx, params) + unit, startTime, endTime, err = function.QueryTimestamp(queryTs.Start, queryTs.End) + metadata.GetQueryParams(ctx).SetTime(startTime, endTime, unit) // 写入查询时间到全局缓存 - metadata.GetQueryParams(ctx).SetTime(start, end, unit) + queryRef, err = queryTs.ToQueryReference(ctx) return } diff --git a/pkg/unify-query/service/http/handler_test.go b/pkg/unify-query/service/http/handler_test.go index 0a54109e0..a9b4574d1 100644 --- a/pkg/unify-query/service/http/handler_test.go +++ b/pkg/unify-query/service/http/handler_test.go @@ -111,8 +111,10 @@ func TestAPIHandler(t *testing.T) { end := time.Unix(1729859485, 0) start := time.Unix(1729863085, 0) + end2d := start.Add(time.Hour * 24 * 2) mock.Vm.Set(map[string]any{ + // `label_values:17298630851729859485container{bcs_cluster_id="BCS-K8S-00000", namespace="kube-system", result_table_id="2_bcs_prom_computation_result_table", __name__="container_cpu_usage_seconds_total_value"}`: []string{ "POD", "kube-proxy", @@ -120,6 +122,17 @@ func TestAPIHandler(t *testing.T) { `label_values:17298630851729859485bcs_cluster_id{bcs_cluster_id="BCS-K8S-00000", result_table_id="2_bcs_prom_computation_result_table", __name__="kube_pod_info_value"}`: []string{ "BCS-K8S-00000", }, + + // above 1d bcs_cluster_id + `label_values:17298630851730035885bcs_cluster_id{result_table_id="2_bcs_prom_computation_result_table", __name__=~"container_.*_value"}`: []string{ + "BCS-K8S-00000", + }, + // above 1d namespace + `label_values:17298630851730035885namespace{result_table_id="2_bcs_prom_computation_result_table", __name__=~"container_.*_value"}`: []string{ + "POD", + "kube-proxy", + }, + `query_range:1729863085172985948560topk(2, count({result_table_id="2_bcs_prom_computation_result_table"}) by (__name__))`: victoriaMetrics.Data{ ResultType: victoriaMetrics.MatrixType, Result: []victoriaMetrics.Series{ @@ -170,6 +183,7 @@ func TestAPIHandler(t *testing.T) { }, }, }, + // below 1d bcs_cluster_id `query_range:1729863085172985948560topk(5, count({result_table_id="2_bcs_prom_computation_result_table", __name__=~"container_.*_value"}) by (bcs_cluster_id))`: victoriaMetrics.Data{ ResultType: victoriaMetrics.MatrixType, Result: []victoriaMetrics.Series{ @@ -185,6 +199,7 @@ func TestAPIHandler(t *testing.T) { }, }, }, + // below 1d namespace `query_range:1729863085172985948560topk(5, count({result_table_id="2_bcs_prom_computation_result_table", __name__=~"container_.*_value"}) by (namespace))`: victoriaMetrics.Data{ ResultType: victoriaMetrics.MatrixType, Result: []victoriaMetrics.Series{ @@ -359,7 +374,7 @@ func TestAPIHandler(t *testing.T) { }, expected: `{"values":{"bcs_cluster_id":["BCS-K8S-00000"],"namespace":["bkbase","bkmonitor-operator","blueking","kube-system"]}}`, }, - "test tag values in prometheus with regex": { + "test tag values in prometheus with regex below 1d": { handler: HandlerTagValues, method: http.MethodPost, infoParams: &infos.Params{ @@ -373,6 +388,20 @@ func TestAPIHandler(t *testing.T) { }, expected: `{"values":{"bcs_cluster_id":["BCS-K8S-00000"],"namespace":["bkbase","bkmonitor-operator","blueking","kube-system"]}}`, }, + "test tag values in prometheus with regex above 1d": { + handler: HandlerTagValues, + method: http.MethodPost, + infoParams: &infos.Params{ + TableID: "result_table.vm", + Start: fmt.Sprintf("%d", start.Unix()), + End: fmt.Sprintf("%d", end2d.Unix()), + IsRegexp: true, + Metric: "container_.*", + Limit: 5, + Keys: []string{"namespace", "bcs_cluster_id"}, + }, + expected: `{"values":{"bcs_cluster_id":["BCS-K8S-00000"],"namespace":["POD","kube-proxy"]}}`, + }, "test series in prometheus": { handler: HandlerSeries, method: http.MethodPost, From 6d92f2ff8571336607a2ad0172b2b537cb3ecd70 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 14 Apr 2025 14:19:39 +0800 Subject: [PATCH 075/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance.go | 58 +++++++++++---------- pkg/unify-query/tsdb/bksql/instance_test.go | 20 +++++++ pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 2 + 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 6f2e712b1..823d20643 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -179,9 +179,6 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad return res, fmt.Errorf("SelectFieldsOrder is empty") } - // 获取该指标的维度 key - dimensions := i.dims(keys, query.Field) - // 获取 metricLabel metricLabel := query.MetricLabels(ctx) @@ -202,14 +199,36 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad continue } - // 获取时间戳，单位是毫秒 - if vtLong, ok = d[sqlExpr.TimeStamp]; !ok { - vtLong = start.UnixMilli() + lbl := make([]prompb.Label, 0) + for _, k := range keys { + switch k { + case sqlExpr.TimeStamp: + if _, ok = d[k]; ok { + vtLong = d[k] + } + case sqlExpr.Value: + if _, ok = d[k]; ok { + vvDouble = d[k] + } + default: + // 获取维度信息 + val, err := getValue(k, d) + if err != nil { + log.Errorf(ctx, "get dimension (%s) value error in %+v %s", k, d, err.Error()) + continue + } + + lbl = append(lbl, prompb.Label{ + Name: k, + Value: val, + }) + } } if vtLong == nil { - continue + vtLong = start.UnixMilli() } + switch vtLong.(type) { case int64: vt = vtLong.(int64) @@ -219,11 +238,6 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad return res, fmt.Errorf("%s type is error %T, %v", dtEventTimeStamp, vtLong, vtLong) } - // 获取值 - if vvDouble, ok = d[sqlExpr.Value]; !ok { - return res, fmt.Errorf("dimension %s is emtpy", sqlExpr.Value) - } - if vvDouble == nil { continue } @@ -236,20 +250,6 @@ func (i *Instance) formatData(ctx context.Context, start time.Time, query *metad return res, fmt.Errorf("%s type is error %T, %v", sqlExpr.Value, vvDouble, vvDouble) } - lbl := make([]prompb.Label, 0) - // 获取维度信息 - for _, dimName := range dimensions { - val, err := getValue(dimName, d) - if err != nil { - return res, fmt.Errorf("dimensions %+v %s", dimensions, err.Error()) - } - - lbl = append(lbl, prompb.Label{ - Name: dimName, - Value: val, - }) - } - // 如果是非时间聚合计算，则无需进行指标名的拼接作用 if metricLabel != nil { lbl = append(lbl, *metricLabel) @@ -311,6 +311,10 @@ func (i *Instance) getFieldsMap(ctx context.Context, sql string) (map[string]str if !ok { continue } + + // 忽略大小写，所以统一转成小写 + k = strings.ToLower(k) + fieldsMap[k] = v } @@ -428,7 +432,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star dataCh <- list } - total = int64(data.TotalRecords) + total = int64(data.TotalRecordSize) return } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index cd6882c8d..0542ef793 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -73,6 +73,10 @@ func TestInstance_QuerySeriesSet(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ + // doris + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":20,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":9,"bkbase_apigw_api":29},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c1","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c2","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":7,"connect_db":66,"match_query_routing_rule":0,"check_permission":72,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":13168,"timetaken":0.152,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", @@ -90,6 +94,22 @@ func TestInstance_QuerySeriesSet(t *testing.T) { query *metadata.Query expected string }{ + "count by cloudId with doris": { + query: &metadata.Query{ + DataSource: datasource, + TableID: tableID, + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + Field: "cloudId", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"cloudId"}, + }, + }, + }, + expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:"},{"name":"cloudId","value":"0"}],"samples":[{"value":6,"timestamp":1730118589181}],"exemplars":null,"histograms":null}]`, + }, "count by namespace with mysql": { query: &metadata.Query{ DataSource: datasource, diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 77fd2b366..95cd9d609 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -359,6 +359,8 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) func (d *DorisSQLExpr) checkMatchALL(k string) bool { if d.fieldsMap != nil { + // 忽略大小写，所以统一转成小写 + k = strings.ToLower(k) if t, ok := d.fieldsMap[k]; ok { if t == DorisTypeText { return true From 5a38f324590be865deaafa190895810c7d123f7a Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 14 Apr 2025 14:30:51 +0800 Subject: [PATCH 076/104] =?UTF-8?q?feat:=20=E6=9F=A5=E8=AF=A2=E5=8E=9F?= =?UTF-8?q?=E5=A7=8B=E6=95=B0=E6=8D=AE=E7=A7=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=20#1010158081123461236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/handler.go | 17 ++++++++++++++--- pkg/unify-query/service/http/query.go | 1 - 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index 5ead655e3..afd2655b9 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -21,6 +21,7 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" ) // HandlerPromQLToStruct @@ -238,7 +239,8 @@ func HandlerQueryRaw(c *gin.Context) { span.Set("query-body", string(queryStr)) listData.TraceID = span.TraceID() - listData.Total, listData.List, listData.ResultTableOptions, err = queryRawWithInstance(ctx, queryTs) + + total, list, resultTableOptions, err := queryRawWithInstance(ctx, queryTs) if err != nil { listData.Status = &metadata.Status{ Code: metadata.QueryRawError, @@ -246,9 +248,18 @@ func HandlerQueryRaw(c *gin.Context) { } return } - if listData.List == nil { - listData.List = make([]map[string]any, 0) + listData.Total = total + listData.List = make([]map[string]any, 0, len(list)) + for _, item := range list { + if item == nil { + continue + } + + delete(item, elasticsearch.KeyAddress) + listData.List = append(listData.List, item) } + listData.ResultTableOptions = resultTableOptions + resp.success(ctx, listData) } diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index b331a2856..352a5ece9 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -288,7 +288,6 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } else { *option.From++ } - } } else { list = list[queryTs.From : queryTs.From+queryTs.Limit] From 2c41ce3aac409a22df665a838c506f1cc2c66cde Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 14 Apr 2025 14:35:00 +0800 Subject: [PATCH 077/104] =?UTF-8?q?feat:=20=E6=9F=A5=E8=AF=A2=E5=8E=9F?= =?UTF-8?q?=E5=A7=8B=E6=95=B0=E6=8D=AE=E7=A7=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=20#1010158081123461236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/handler.go | 17 ++++++++++------- .../tsdb/elasticsearch/instance_test.go | 3 --- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index afd2655b9..b9b9f58bc 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -199,12 +199,13 @@ func HandlerQueryExemplar(c *gin.Context) { // @Router /query/raw [post] func HandlerQueryRaw(c *gin.Context) { var ( - ctx = c.Request.Context() - resp = &response{c: c} - user = metadata.GetUser(ctx) - err error - span *trace.Span - listData ListData + ctx = c.Request.Context() + resp = &response{c: c} + user = metadata.GetUser(ctx) + err error + span *trace.Span + listData ListData + ignoreDimensions = []string{elasticsearch.KeyAddress} ) ctx, span = trace.NewSpan(ctx, "handler-query-raw") @@ -255,7 +256,9 @@ func HandlerQueryRaw(c *gin.Context) { continue } - delete(item, elasticsearch.KeyAddress) + for _, ignoreDimension := range ignoreDimensions { + delete(item, ignoreDimension) + } listData.List = append(listData.List, item) } listData.ResultTableOptions = resultTableOptions diff --git a/pkg/unify-query/tsdb/elasticsearch/instance_test.go b/pkg/unify-query/tsdb/elasticsearch/instance_test.go index 0e4e8a706..fd2c90d12 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance_test.go @@ -485,9 +485,6 @@ func TestInstance_queryRawData(t *testing.T) { // nested query + query string 测试 + highlight `{"_source":{"includes":["group","user.first","user.last"]},"from":0,"highlight":{"fields":{"*":{}},"number_of_fragments":0,"post_tags":["\u003c/mark\u003e"],"pre_tags":["\u003cmark\u003e"],"require_field_match":true},"query":{"bool":{"filter":[{"nested":{"path":"user","query":{"match_phrase":{"user.first":{"query":"John"}}}}},{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}},{"query_string":{"analyze_wildcard":true,"fields":["*","__*"],"lenient":true,"query":"group: fans"}}]}},"size":5,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.0,"hits":[{"_index":"bk_unify_query_demo_2","_type":"_doc","_id":"aS3KjpEBbwEm76LbcH1G","_score":0.0,"_source":{"user":[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}],"group":"fans"},"highlight":{"user.new_group_user_first":["John"],"user.first":["John"],"user.new_first":["John"],"group":["fans"]}}]}}`, - - - // "nested aggregate + query 测试 `{"_source":{"includes":["group","user.first","user.last"]},"aggregations":{"user":{"aggregations":{"_value":{"value_count":{"field":"user.first"}}},"nested":{"path":"user"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723593608,"include_lower":true,"include_upper":true,"to":1723679962}}}}},"size":0}`: `{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":17,"relation":"eq"},"max_score":null,"hits":[]},"aggregations":{"user":{"doc_count":18,"_value":{"value":18}}}}`, From 0a92a14209b65fe06f905fe2375daf722586a02c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 14 Apr 2025 22:33:25 +0800 Subject: [PATCH 078/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/json/custom.go | 33 ++++++ pkg/unify-query/internal/json/custom_test.go | 112 +++++++++++++++++++ pkg/unify-query/tsdb/bksql/format.go | 29 +++++ pkg/unify-query/tsdb/bksql/instance.go | 8 +- pkg/unify-query/tsdb/bksql/instance_test.go | 3 +- 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 pkg/unify-query/internal/json/custom_test.go diff --git a/pkg/unify-query/internal/json/custom.go b/pkg/unify-query/internal/json/custom.go index 01979ae13..675a4b19d 100644 --- a/pkg/unify-query/internal/json/custom.go +++ b/pkg/unify-query/internal/json/custom.go @@ -10,11 +10,44 @@ package json import ( + "encoding/json" "fmt" "sort" "strings" ) +const ( + StepString = "." +) + +func mapData(prefix string, data map[string]any, res map[string]any) { + for k, v := range data { + if prefix != "" { + k = prefix + StepString + k + } + switch v.(type) { + case map[string]any: + mapData(k, v.(map[string]any), res) + default: + res[k] = v + } + } +} + +// ParseObject 解析 json，按照层级打平 +func ParseObject(prefix, intput string) (map[string]any, error) { + oldData := make(map[string]any) + newData := make(map[string]any) + + err := json.Unmarshal([]byte(intput), &oldData) + if err != nil { + return newData, err + } + + mapData(prefix, oldData, newData) + return newData, nil +} + func MarshalListMap(data []map[string]interface{}) string { if len(data) == 0 { return "[]" diff --git a/pkg/unify-query/internal/json/custom_test.go b/pkg/unify-query/internal/json/custom_test.go new file mode 100644 index 000000000..8a9c3f5f4 --- /dev/null +++ b/pkg/unify-query/internal/json/custom_test.go @@ -0,0 +1,112 @@ +// Tencent is pleased to support the open source community by making +// 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package json_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" +) + +func TestParseJson(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]any + wantErr bool + }{ + { + name: "test1", + input: `{"a": "b"}`, + expected: map[string]any{ + "a": "b", + }, + }, + { + name: "normal nested json", + input: `{"a": {"b": 1, "c": "test"}, "d": true}`, + expected: map[string]interface{}{ + "a.b": float64(1), + "a.c": "test", + "d": true, + }, + wantErr: false, + }, + { + name: "single level json", + input: `{"key1": "value1", "key2": 123}`, + expected: map[string]interface{}{ + "key1": "value1", + "key2": float64(123), // JSON numbers are decoded as float64 + }, + wantErr: false, + }, + { + name: "empty json", + input: `{}`, + expected: map[string]interface{}{}, + wantErr: false, + }, + { + name: "invalid json", + input: `{"key": "value"`, + expected: map[string]interface{}{}, + wantErr: true, + }, + { + name: "json with special characters in keys", + input: `{"a.b": {"c-d": "value"}}`, + expected: map[string]interface{}{ + "a.b.c-d": "value", + }, + wantErr: false, + }, + { + name: "deeply nested json", + input: `{"a": {"b": {"c": {"d": "value"}}}}`, + expected: map[string]interface{}{ + "a.b.c.d": "value", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := json.ParseObject("__ext", tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseObject() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + assert.Equal(t, got, tt.expected) + } + }) + } +} + +func TestParseJson_WithArrays(t *testing.T) { + // 虽然当前实现不支持数组，但测试一下看看行为 + input := `{"a": [1, 2, 3], "b": {"c": [4, 5]}}` + expected := map[string]any{ + "a": []any{float64(1), float64(2), float64(3)}, + "b.c": []any{float64(4), float64(5)}, + } + got, err := json.ParseObject("__ext", input) + if err != nil { + t.Errorf("ParseObject() with arrays returned error: %v", err) + return + } + + assert.Equal(t, got, expected) +} diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 319ca5e11..a77a7a802 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -20,6 +20,8 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/json" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/log" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/bksql/sqlExpr" @@ -121,6 +123,33 @@ func (f *QueryFactory) FieldMap() map[string]string { return f.expr.FieldMap() } +func (f *QueryFactory) ReloadListData(data map[string]any) map[string]any { + newData := make(map[string]any) + newData[KeyIndex] = f.query.DB + newData[KeyTableID] = f.query.TableID + newData[KeyDataLabel] = f.query.DataLabel + + fieldMap := f.FieldMap() + for k, d := range data { + if v, ok := fieldMap[k]; ok { + if v == TableTypeVariant { + objectData, err := json.ParseObject(k, d.(string)) + if err != nil { + log.Errorf(f.ctx, "json.ParseObject err: %v", err) + continue + } + for nk, nd := range objectData { + newData[nk] = nd + } + continue + } + } + + newData[k] = d + } + return newData +} + func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 823d20643..736e7e591 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -42,6 +42,8 @@ const ( TableFieldName = "Field" TableFieldType = "Type" + + TableTypeVariant = "variant" ) type Instance struct { @@ -425,11 +427,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } for _, list := range data.List { - list[KeyIndex] = query.DB - list[KeyTableID] = query.TableID - list[KeyDataLabel] = query.DataLabel - - dataCh <- list + dataCh <- queryFactory.ReloadListData(list) } total = int64(data.TotalRecordSize) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 0542ef793..4ea87fb6b 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -173,6 +173,7 @@ func TestInstance_QueryRaw(t *testing.T) { mock.BkSQL.Set(map[string]any{ "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) @@ -220,7 +221,7 @@ func TestInstance_QueryRaw(t *testing.T) { DataLabel: "5000140_bklog_container_log_demo_analysis", Size: 2, }, - expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext":"{\"container_id\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"container_image\":\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\",\"container_name\":\"bkmonitorbeat\",\"io_kubernetes_pod\":\"bkm-daemonset-worker-9tckj\",\"io_kubernetes_pod_namespace\":\"bkmonitor-operator\",\"io_kubernetes_pod_uid\":\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\",\"io_kubernetes_workload_name\":\"bkm-daemonset-worker\",\"io_kubernetes_workload_type\":\"DaemonSet\",\"labels\":{\"app_kubernetes_io_component\":\"bkmonitorbeat\",\"controller_revision_hash\":\"6b87cb95fc\",\"pod_template_generation\":\"14\"}}","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext":"{\"container_id\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"container_image\":\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\",\"container_name\":\"bkmonitorbeat\",\"io_kubernetes_pod\":\"bkm-daemonset-worker-9tckj\",\"io_kubernetes_pod_namespace\":\"bkmonitor-operator\",\"io_kubernetes_pod_uid\":\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\",\"io_kubernetes_workload_name\":\"bkm-daemonset-worker\",\"io_kubernetes_workload_type\":\"DaemonSet\",\"labels\":{\"app_kubernetes_io_component\":\"bkmonitorbeat\",\"controller_revision_hash\":\"6b87cb95fc\",\"pod_template_generation\":\"14\"}}","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, }, } { t.Run(name, func(t *testing.T) { From 9a4c4da65f2aacad7abf2976a230fbae28ddacf4 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 14 Apr 2025 22:36:35 +0800 Subject: [PATCH 079/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/json/custom_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/unify-query/internal/json/custom_test.go b/pkg/unify-query/internal/json/custom_test.go index 8a9c3f5f4..8e1584ead 100644 --- a/pkg/unify-query/internal/json/custom_test.go +++ b/pkg/unify-query/internal/json/custom_test.go @@ -28,16 +28,16 @@ func TestParseJson(t *testing.T) { name: "test1", input: `{"a": "b"}`, expected: map[string]any{ - "a": "b", + "__ext.a": "b", }, }, { name: "normal nested json", input: `{"a": {"b": 1, "c": "test"}, "d": true}`, expected: map[string]interface{}{ - "a.b": float64(1), - "a.c": "test", - "d": true, + "__ext.a.b": float64(1), + "__ext.a.c": "test", + "__ext.d": true, }, wantErr: false, }, @@ -45,8 +45,8 @@ func TestParseJson(t *testing.T) { name: "single level json", input: `{"key1": "value1", "key2": 123}`, expected: map[string]interface{}{ - "key1": "value1", - "key2": float64(123), // JSON numbers are decoded as float64 + "__ext.key1": "value1", + "__ext.key2": float64(123), // JSON numbers are decoded as float64 }, wantErr: false, }, @@ -66,7 +66,7 @@ func TestParseJson(t *testing.T) { name: "json with special characters in keys", input: `{"a.b": {"c-d": "value"}}`, expected: map[string]interface{}{ - "a.b.c-d": "value", + "__ext.a.b.c-d": "value", }, wantErr: false, }, @@ -74,7 +74,7 @@ func TestParseJson(t *testing.T) { name: "deeply nested json", input: `{"a": {"b": {"c": {"d": "value"}}}}`, expected: map[string]interface{}{ - "a.b.c.d": "value", + "__ext.a.b.c.d": "value", }, wantErr: false, }, @@ -99,8 +99,8 @@ func TestParseJson_WithArrays(t *testing.T) { // 虽然当前实现不支持数组，但测试一下看看行为 input := `{"a": [1, 2, 3], "b": {"c": [4, 5]}}` expected := map[string]any{ - "a": []any{float64(1), float64(2), float64(3)}, - "b.c": []any{float64(4), float64(5)}, + "__ext.a": []any{float64(1), float64(2), float64(3)}, + "__ext.b.c": []any{float64(4), float64(5)}, } got, err := json.ParseObject("__ext", input) if err != nil { From 5e1f095b995861dcfcaba6c859f739c57c48400a Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 15 Apr 2025 10:39:46 +0800 Subject: [PATCH 080/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 11 +++++------ pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 95cd9d609..2b9b12dea 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -23,6 +23,8 @@ const ( DorisTypeText = "text" ShardKey = "__shard_key__" + + DefaultKey = "log" ) type DorisSQLExpr struct { @@ -406,8 +408,7 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return fmt.Sprintf("%s AND %s", left, right), nil case *querystring.WildcardExpr: if c.Field == "" { - err = fmt.Errorf(Doris + " " + ErrorMatchAll) - return "", err + c.Field = DefaultKey } field, _ := d.dimTransform(c.Field) @@ -415,8 +416,7 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil case *querystring.MatchExpr: if c.Field == "" { - err = fmt.Errorf(Doris + " " + ErrorMatchAll + ": " + c.Value) - return "", err + c.Field = DefaultKey } field, _ := d.dimTransform(c.Field) @@ -427,8 +427,7 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { return fmt.Sprintf("%s = '%s'", field, c.Value), nil case *querystring.NumberRangeExpr: if c.Field == "" { - err = fmt.Errorf(Doris + " " + ErrorMatchAll) - return "", err + c.Field = DefaultKey } field, _ := d.dimTransform(c.Field) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 2c59428af..17b28e403 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -34,7 +34,8 @@ func TestDorisSQLExpr_ParserQueryString(t *testing.T) { { name: "one word", input: "test", - err: "doris 不支持全字段检索: test", + want: "`log` = 'test'", + //err: "doris 不支持全字段检索: test", }, { name: "complex nested query", From 7dff4ccd70f6819e91e17438fb68b39418353187 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 15 Apr 2025 14:25:35 +0800 Subject: [PATCH 081/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/internal/function/function.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/unify-query/internal/function/function.go b/pkg/unify-query/internal/function/function.go index f63d8a10e..13acd141f 100644 --- a/pkg/unify-query/internal/function/function.go +++ b/pkg/unify-query/internal/function/function.go @@ -13,7 +13,6 @@ import ( "fmt" "math" "strconv" - "strings" "time" "github.com/prometheus/prometheus/model/labels" @@ -26,19 +25,6 @@ const ( Nanosecond = "nanosecond" ) -func MatcherToString(matchers ...*labels.Matcher) string { - vals := make([]string, 0, len(matchers)) - for _, m := range matchers { - vals = append(vals, m.String()) - } - - if len(vals) == 0 { - return "" - } - - return fmt.Sprintf("{%s}", strings.Join(vals, ",")) -} - func MatcherToMetricName(matchers ...*labels.Matcher) string { for _, m := range matchers { if m.Name == labels.MetricName { From bcd6740e3c29b78a11b071c4deddb2aef7e8d0ea Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 15 Apr 2025 14:59:21 +0800 Subject: [PATCH 082/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/query_ts.go | 1 + pkg/unify-query/tsdb/bksql/format.go | 2 +- pkg/unify-query/tsdb/bksql/format_test.go | 33 ++++++++-- pkg/unify-query/tsdb/bksql/instance_test.go | 68 ++++++++++---------- 4 files changed, 62 insertions(+), 42 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 3fac3360a..ca5a55782 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -625,6 +625,7 @@ func (q *Query) ToQueryMetric(ctx context.Context, spaceUid string) (*metadata.Q span.Set("query-metric", metricName) span.Set("query-is-regexp", q.IsRegexp) span.Set("tsdb-num", len(tsDBs)) + span.Set("query-aggregate", aggregates) for _, tsDB := range tsDBs { query, buildErr := q.BuildMetadataQuery(ctx, tsDB, allConditions, queryLabelsMatcher) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index a77a7a802..6e1fde546 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -177,7 +177,7 @@ func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error func (f *QueryFactory) BuildWhere() (string, error) { var s []string - s = append(s, fmt.Sprintf("`%s` >= %d AND `%s` < %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) + s = append(s, fmt.Sprintf("`%s` >= %d AND `%s` <= %d", f.timeField, f.start.UnixMilli(), f.timeField, f.end.UnixMilli())) theDateFilter, err := f.getTheDateIndexFilters() if err != nil { diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 08ccb3361..21d89dabf 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -71,7 +71,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-count_over_time-with-promql-seconds": { query: &metadata.Query{ @@ -110,7 +110,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 75000 AS INT) * 75000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", + expected: "SELECT `level`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 75000 AS INT) * 75000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 AND `level` = 'ERROR' GROUP BY `level`, _timestamp_ ORDER BY `_timestamp_` ASC, `level` ASC", }, "doris sum-with-promql-1": { query: &metadata.Query{ @@ -138,7 +138,7 @@ func TestNewSqlFactory(t *testing.T) { Size: 10, Orders: nil, }, - expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", + expected: "SELECT `ip`, SUM(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' AND `gseIndex` > 0 GROUP BY `ip` LIMIT 10", }, "doris count-with-count-promql-1": { query: &metadata.Query{ @@ -155,7 +155,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` < 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1741795260000 AND `dtEventTimeStamp` <= 1741796260000 AND `thedate` = '20250313' GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count-with-count-promql-2": { // 2024-12-07 21:36:40 UTC @@ -187,7 +187,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` < 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1733607400000 AND `dtEventTimeStamp` <= 1733939375000 AND `thedate` >= '20241208' AND `thedate` <= '20241212' AND `gseIndex` > 0 GROUP BY `ip`, _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count by day with UTC": { // 2025-03-14 15:05:45 Asia/ShangHai @@ -205,7 +205,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", }, "doris count by day with Asia/Shanghai": { // 2025-03-14 15:05:45 Asia/ShangHai @@ -224,7 +224,26 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 480) / 1440 AS INT) * 1440 - 480) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` < 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 480) / 1440 AS INT) * 1440 - 480) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + }, + "doris count by dimension with object": { + // 2025-03-14 15:05:45 Asia/ShangHai + start: time.UnixMilli(1741935945000), + // 2025-03-20 15:35:45 Asia/ShangHai + end: time.UnixMilli(1742456145000), + query: &metadata.Query{ + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + Field: "__ext.container_id", + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Hour * 24, + Dimensions: []string{"__ext.container_id"}, + }, + }, + }, + expected: "SELECT CAST(__ext[\"container_id\"] AS STRING) AS `__ext__bk_46__container_id`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY CAST(__ext[\"container_id\"] AS STRING), _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 4ea87fb6b..adfe34b30 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -75,11 +75,11 @@ func TestInstance_QuerySeriesSet(t *testing.T) { mock.BkSQL.Set(map[string]any{ // doris "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":20,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":9,"bkbase_apigw_api":29},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c1","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c2","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":7,"connect_db":66,"match_query_routing_rule":0,"check_permission":72,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":13168,"timetaken":0.152,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, - "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", - "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) end := time.UnixMilli(1730118889181) @@ -174,8 +174,8 @@ func TestInstance_QueryRaw(t *testing.T) { mock.BkSQL.Set(map[string]any{ "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", - "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` < 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) end := time.UnixMilli(1730118889181) @@ -294,7 +294,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 15000 * 15000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (dtEventTimeStamp + 0) / 15000 * 15000 - 0 ORDER BY `_timestamp_` ASC", + expected: "SELECT `namespace`, COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 15000 * 15000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND `namespace` IN ('bgp2-new', 'gz100') GROUP BY `namespace`, (dtEventTimeStamp + 0) / 15000 * 15000 - 0 ORDER BY `_timestamp_` ASC", }, { name: "conditions with or", @@ -329,7 +329,7 @@ func TestInstance_bkSql(t *testing.T) { }, BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test'", }, { name: "conditions with or and", @@ -387,7 +387,7 @@ func TestInstance_bkSql(t *testing.T) { }, BkSqlCondition: "(`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` NOT LIKE '%test%' AND `namespace` NOT LIKE '%test2%') AND `namespace` NOT IN ('test', 'test2') AND `namespace` NOT LIKE '%test%' AND `namespace` != 'test' AND (`text` NOT LIKE '%test%' AND `text` NOT LIKE '%test2%') AND (`text` NOT MATCH_PHRASE_PREFIX 'test' AND `text` NOT MATCH_PHRASE_PREFIX 'test2') AND `text` NOT LIKE '%test%' AND `text` NOT MATCH_PHRASE_PREFIX 'test'", }, { name: "conditions with or and like", @@ -422,7 +422,7 @@ func TestInstance_bkSql(t *testing.T) { }, BkSqlCondition: "(`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, - expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", + expected: "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND (`namespace` LIKE '%test%' OR `namespace` LIKE '%test2%') AND `namespace` IN ('test', 'test2') AND `namespace` LIKE '%test%' AND `namespace` = 'test'", }, { name: "aggregate sum", @@ -436,7 +436,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT SUM(`value`) AS `_value_` FROM `132_hander_opmon_avg` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { name: "aggregate cardinality with mysql", @@ -465,7 +465,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { name: "aggregate date_histogram with mysql", @@ -498,7 +498,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`gseIndex`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 300000 * 300000 - 0) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY (dtEventTimeStamp + 0) / 300000 * 300000 - 0 ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 300000 * 300000 - 0) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY (dtEventTimeStamp + 0) / 300000 * 300000 - 0 ORDER BY `_timestamp_` ASC", }, { name: "aggregate cardinality with doris", @@ -527,7 +527,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT COUNT(DISTINCT `gseIndex`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, { name: "aggregate date_histogram with doris", @@ -560,7 +560,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`gseIndex`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC", }, //{ @@ -607,7 +607,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `dtEventTimeStamp` DESC, `gseIndex` DESC, `iterationIndex` DESC LIMIT 5", + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `dtEventTimeStamp` DESC, `gseIndex` DESC, `iterationIndex` DESC LIMIT 5", }, { name: "query raw", @@ -617,7 +617,7 @@ func TestInstance_bkSql(t *testing.T) { Field: "value", Size: 5, }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' LIMIT 5", + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' LIMIT 5", }, { name: "query raw with order desc", @@ -632,7 +632,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `_timestamp_` DESC", + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `_timestamp_` DESC", }, { name: "query aggregate count and dimensions", @@ -651,7 +651,7 @@ func TestInstance_bkSql(t *testing.T) { Size: 5, }, - expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' GROUP BY `ip` LIMIT 5", + expected: "SELECT `ip`, COUNT(`gseIndex`) AS `_value_` FROM `100133_ieod_logsearch4_errorlog_p`.doris WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' GROUP BY `ip` LIMIT 5", }, { name: "query aggregate count", @@ -667,7 +667,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", }, { name: "query aggregate count with window hour", @@ -684,7 +684,7 @@ func TestInstance_bkSql(t *testing.T) { }, }, - expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 3600000 * 3600000 - 0) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (dtEventTimeStamp + 0) / 3600000 * 3600000 - 0 ORDER BY `_timestamp_` ASC", + expected: "SELECT COUNT(`matchstep_start_to_fail_0_100`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 3600000 * 3600000 - 0) AS `_timestamp_` FROM `101068_MatchFullLinkTimeConsumptionFlow_CostTime` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210' GROUP BY (dtEventTimeStamp + 0) / 3600000 * 3600000 - 0 ORDER BY `_timestamp_` ASC", }, } @@ -742,7 +742,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { DB: "test_db", Field: "value", }, - expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `test_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT *, `value` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `test_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, // 测试用例2: 多聚合函数组合 @@ -756,7 +756,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { {Name: "min"}, }, }, - expected: "SELECT MAX(`temperature`) AS `_value_`, MIN(`temperature`) AS `_value_` FROM `metrics_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT MAX(`temperature`) AS `_value_`, MIN(`temperature`) AS `_value_` FROM `metrics_db` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, // 测试用例3: 复杂条件组合 @@ -780,7 +780,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, }, }, - expected: "SELECT *, `duration` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `security_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' AND `severity` IN ('high', 'critical') AND `source_ip` != '192.168.1.1'", + expected: "SELECT *, `duration` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `security_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' AND `severity` IN ('high', 'critical') AND `source_ip` != '192.168.1.1'", }, // 测试用例4: 多字段排序 @@ -799,7 +799,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, }, }, - expected: "SELECT *, `amount` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612' ORDER BY `account_id` DESC, `timestamp` ASC", + expected: "SELECT *, `amount` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `transaction_logs` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612' ORDER BY `account_id` DESC, `timestamp` ASC", }, // 测试用例5: 特殊字符转义 @@ -813,7 +813,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { {Name: "sum"}, }, }, - expected: "SELECT SUM(`*`) AS `_value_` FROM `special_metrics`.select WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT SUM(`*`) AS `_value_` FROM `special_metrics`.select WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, // 测试用例6: 零窗口时间 @@ -829,7 +829,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, }, }, - expected: "SELECT AVG(`value`) AS `_value_` FROM `time_series_data` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` < 1718193555000 AND `thedate` = '20240612'", + expected: "SELECT AVG(`value`) AS `_value_` FROM `time_series_data` WHERE `dtEventTimeStamp` >= 1718189940000 AND `dtEventTimeStamp` <= 1718193555000 AND `thedate` = '20240612'", }, // 测试用例7: 跨多天的时间范围 @@ -844,7 +844,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { {Name: "count"}, }, }, - expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` < 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", + expected: "SELECT COUNT(`active_users`) AS `_value_` FROM `daily_metrics` WHERE `dtEventTimeStamp` >= 1733756400000 AND `dtEventTimeStamp` <= 1733846399000 AND `thedate` >= '20241209' AND `thedate` <= '20241210'", }, // 测试用例8: 默认处理 object 字段 @@ -917,7 +917,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", }, // 测试用例10: doris 处理 object 字段 + 时间聚合 { @@ -956,7 +956,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例11: doris 处理 object 字段 + 时间聚合 5m { @@ -982,7 +982,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例12: doris 处理 object 字段 + 时间聚合 15s { @@ -1008,7 +1008,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` < 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, } @@ -1042,7 +1042,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { // 验证时间条件 if tc.start.IsZero() && tc.end.IsZero() { assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` >= %d", baseStart.UnixMilli())) - assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` < %d", baseEnd.UnixMilli())) + assert.Contains(t, generatedSQL, fmt.Sprintf("`dtEventTimeStamp` <= %d", baseEnd.UnixMilli())) } } } @@ -1062,7 +1062,7 @@ func TestInstance_QueryLabelNames_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, + "SELECT *, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` <= 1740553771000 AND `thedate` = '20250226' LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":1,"external_api_call_time_mills":{"bkbase_meta_api":8},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250226,"dteventtimestamp":1740552000000,"dteventtime":"2025-02-26 14:40:00","localtime":"2025-02-26 14:45:01","_starttime_":"2025-02-26 14:40:00","_endtime_":"2025-02-26 14:40:00","bk_host_id":5279498,"__ext":"{\"container_id\":\"101e58e9940c78a374e4ca3fe28d2360a8dd38b5b93937f7996902c203ac7812\",\"container_name\":\"ds\",\"bk_bcs_cluster_id\":\"BCS-K8S-26678\",\"io_kubernetes_pod\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"container_image\":\"proz-tcr.tencentcloudcr.com/a1_proz/proz-ds@sha256:0ccc969d0614c41e9418ab81f444a26db743e82d3a2a2cc2d12e549391c5768f\",\"io_kubernetes_pod_namespace\":\"ds9204\",\"io_kubernetes_workload_type\":\"GameServer\",\"io_kubernetes_pod_uid\":\"78e5a0cf-fdec-43aa-9c64-5e58c35c949d\",\"io_kubernetes_workload_name\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"labels\":{\"agones_dev_gameserver\":\"ds-pro-z-instance-season-p-qvq6l-8fbrq\",\"agones_dev_role\":\"gameserver\",\"agones_dev_safe_to_evict\":\"false\",\"component\":\"ds\",\"part_of\":\"projectz\"}}","cloudid":0,"path":"/proz/LinuxServer/ProjectZ/Saved/Logs/Stats/ObjectStat_ds-pro-z-instance-season-p-qvq6l-8fbrq-0_2025.02.26-04.25.48.368.log","gseindex":1399399185,"iterationindex":185,"log":"[2025.02.26-14.40.00:711][937] BTT_SetLocationWarpTarget_C 35","time":1740552000,"_timestamp_":1740552000000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":54,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":9,"connect_db":34,"match_query_routing_rule":0,"check_permission":9,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","_starttime_","_endtime_","bk_host_id","__ext","cloudid","path","gseindex","iterationindex","log","time","_timestamp_"],"total_record_size":4512,"timetaken":0.107,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"_starttime_","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"_endtime_","field_index":5},{"field_type":"int","field_name":"__c6","field_alias":"bk_host_id","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"__ext","field_index":7},{"field_type":"int","field_name":"__c8","field_alias":"cloudid","field_index":8},{"field_type":"string","field_name":"__c10","field_alias":"path","field_index":10},{"field_type":"long","field_name":"__c11","field_alias":"gseindex","field_index":11},{"field_type":"int","field_name":"__c12","field_alias":"iterationindex","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"long","field_name":"__c14","field_alias":"time","field_index":14},{"field_type":"long","field_name":"__c15","field_alias":"_timestamp_","field_index":15}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3465b590d66a21d3aae7841d36aaec3d","span_id":"34296e9388f3258a"}`, }) // 测试用例 @@ -1114,7 +1114,7 @@ func TestInstance_QueryLabelValues_Normal(t *testing.T) { // mock 查询数据 mock.BkSQL.Set(map[string]any{ "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` < 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, + "SELECT `bk_host_id`, COUNT(*) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1740551971000 AND `dtEventTimeStamp` <= 1740553771000 AND `thedate` = '20250226' GROUP BY `bk_host_id` LIMIT 2": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"5000140_bklog_container_log_demo_analysis":{"start":"2025022600","end":"2025022623"}},"cluster":"doris_bklog","totalRecords":26,"external_api_call_time_mills":{"bkbase_meta_api":6},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"bk_host_id":5843771,"_value_":6520005},{"bk_host_id":4580470,"_value_":703143}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":204,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":39,"match_query_routing_rule":0,"check_permission":6,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["bk_host_id","_value_"],"total_record_size":6952,"timetaken":0.257,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"bk_host_id","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"3592ea81c52ab826aba587d91e5054b6","span_id":"f21eca23481c778d"}`, }) // 测试用例 From 7995135c313e592899ef5ad42647a5483dfc9851 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 16 Apr 2025 17:47:22 +0800 Subject: [PATCH 083/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/influxdb/router_mock.go | 17 +- pkg/unify-query/metadata/struct.go | 111 ++++++------ pkg/unify-query/mock/handler.go | 77 ++++---- pkg/unify-query/query/structured/method.go | 2 +- pkg/unify-query/query/structured/query_ts.go | 2 +- .../query/structured/query_ts_test.go | 7 +- pkg/unify-query/query/structured/space.go | 2 +- .../query/structured/space_test.go | 10 +- pkg/unify-query/service/http/handler.go | 29 +--- pkg/unify-query/service/http/query.go | 29 +++- pkg/unify-query/service/http/query_test.go | 164 ++++++++++++++++-- pkg/unify-query/tsdb/bksql/client_test.go | 2 +- pkg/unify-query/tsdb/bksql/format.go | 131 +++++++++++++- pkg/unify-query/tsdb/bksql/instance.go | 125 +------------ pkg/unify-query/tsdb/bksql/instance_test.go | 4 +- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 102 ++++++----- .../tsdb/elasticsearch/instance.go | 12 +- .../tsdb/victoriaMetrics/instance_test.go | 4 +- 18 files changed, 500 insertions(+), 330 deletions(-) diff --git a/pkg/unify-query/influxdb/router_mock.go b/pkg/unify-query/influxdb/router_mock.go index f57fd0dac..34979c290 100644 --- a/pkg/unify-query/influxdb/router_mock.go +++ b/pkg/unify-query/influxdb/router_mock.go @@ -34,6 +34,7 @@ const ( ResultTableEs = "result_table.es" ResultTableBkBaseEs = "result_table.bk_base_es" ResultTableBkSQL = "result_table.bk_sql" + ResultTableDoris = "result_table.doris" ) var ( @@ -123,7 +124,7 @@ func MockSpaceRouter(ctx context.Context) { ) tsdb.SetStorage("2", &tsdb.Storage{Type: consul.InfluxDBStorageType}) tsdb.SetStorage("3", &tsdb.Storage{Type: consul.ElasticsearchStorageType, Address: mock.EsUrl}) - tsdb.SetStorage("4", &tsdb.Storage{Type: consul.BkSqlStorageType, Address: mock.BkSQLUrl}) + tsdb.SetStorage("4", &tsdb.Storage{Type: consul.BkSqlStorageType, Address: mock.BkBaseUrl}) r := GetInfluxDBRouter() r.clusterInfo = ir.ClusterInfo{ @@ -183,6 +184,9 @@ func MockSpaceRouter(ctx context.Context) { ResultTableBkBaseEs: &ir.SpaceResultTable{ TableId: ResultTableBkBaseEs, }, + ResultTableDoris: &ir.SpaceResultTable{ + TableId: ResultTableDoris, + }, }, }, ir.ResultTableDetailInfo{ @@ -254,7 +258,7 @@ func MockSpaceRouter(ctx context.Context) { StorageId: 3, TableId: ResultTableEs, DB: "es_index", - SourceType: "bkdata", + SourceType: "", StorageType: consul.ElasticsearchStorageType, StorageClusterRecords: []ir.Record{ { @@ -274,6 +278,15 @@ func MockSpaceRouter(ctx context.Context) { StorageId: 4, TableId: ResultTableBkSQL, DataLabel: "bksql", + DB: "2_bklog_bkunify_query_doris", + StorageType: consul.BkSqlStorageType, + }, + ResultTableDoris: &ir.ResultTableDetail{ + StorageId: 4, + TableId: ResultTableDoris, + DB: "2_bklog_bkunify_query_doris", + Measurement: "doris", + DataLabel: "bksql", StorageType: consul.BkSqlStorageType, }, ResultTableBkBaseEs: &ir.ResultTableDetail{ diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 560db9256..192af08c9 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -50,23 +50,23 @@ const ( type VmCondition string type TimeField struct { - Name string - Type string - Unit string + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Unit string `json:"unit,omitempty"` } // Aggregate 聚合方法 type Aggregate struct { - Name string - Field string + Name string `json:"name,omitempty"` + Field string `json:"field,omitempty"` - Dimensions []string - Without bool + Dimensions []string `json:"dimensions,omitempty"` + Without bool `json:"without,omitempty"` - Window time.Duration - TimeZone string + Window time.Duration `json:"window,omitempty"` + TimeZone string `json:"time_zone,omitempty"` - Args []interface{} + Args []interface{} `json:"args,omitempty"` } // OffSetInfo Offset的信息存储，供promql查询转换为influxdb查询语句时使用 @@ -81,73 +81,73 @@ type Aggregates []Aggregate // Query 查询扩展信息，为后面查询提供定位 type Query struct { - SourceType string - Password string // 查询鉴权 + SourceType string `json:"source_type,omitempty"` + Password string `json:"password,omitempty"` // 查询鉴权 - ClusterID string // 存储 ID + ClusterID string `json:"cluster_id,omitempty"` // 存储 ID - StorageType string // 存储类型 + StorageType string `json:"storage_type,omitempty"` // 存储类型 - StorageIDs []string - StorageID string - StorageName string + StorageIDs []string `json:"storage_ids,omitempty"` + StorageID string `json:"storage_id,omitempty"` + StorageName string `json:"storage_name,omitempty"` - ClusterName string - TagsKey []string + ClusterName string `json:"cluster_name,omitempty"` + TagsKey []string `json:"tags_key,omitempty"` - DataSource string - DataLabel string - TableID string - MetricName string + DataSource string `json:"data_source,omitempty"` + DataLabel string `json:"data_label,omitempty"` + TableID string `json:"table_id,omitempty"` + MetricName string `json:"metric_name,omitempty"` // vm 的 rt - VmRt string + VmRt string `json:"vm_rt,omitempty"` // 兼容 InfluxDB 结构体 - RetentionPolicy string // 存储 RP - DB string // 存储 DB - Measurement string // 存储 Measurement - Field string // 存储 Field - TimeField TimeField // 时间字段 - Timezone string // 存储 Timezone - Fields []string // 存储命中的 Field 列表，一般情况下为一个，当 Field 为模糊匹配时，解析为多个 - Measurements []string // 存储命中的 Measurement 列表，一般情况下为一个，当 Measurement 为模糊匹配时，解析为多个 + RetentionPolicy string `json:"retention_policy,omitempty"` // 存储 RP + DB string `json:"db,omitempty"` // 存储 DB + Measurement string `json:"measurement,omitempty"` // 存储 Measurement + Field string `json:"field,omitempty"` // 存储 Field + TimeField TimeField `json:"time_field,omitempty"` // 时间字段 + Timezone string `json:"timezone,omitempty"` // 存储 Timezone + Fields []string `json:"fields,omitempty"` // 存储命中的 Field 列表，一般情况下为一个，当 Field 为模糊匹配时，解析为多个 + Measurements []string `json:"measurements,omitempty"` // 存储命中的 Measurement 列表，一般情况下为一个，当 Measurement 为模糊匹配时，解析为多个 // 用于 promql 查询 - IsHasOr bool // 标记是否有 or 条件 + IsHasOr bool `json:"is_has_or,omitempty"` // 标记是否有 or 条件 - Aggregates Aggregates // 聚合方法列表，从内到外排序 + Aggregates Aggregates `json:"aggregates,omitempty"` // 聚合方法列表，从内到外排序 - Condition string // 过滤条件 + Condition string `json:"condition,omitempty"` // 过滤条件 // BkSql 过滤条件 - BkSqlCondition string + BkSqlCondition string `json:"bk_sql_condition,omitempty"` // Vm 过滤条件 - VmCondition VmCondition - VmConditionNum int + VmCondition VmCondition `json:"vm_condition,omitempty"` + VmConditionNum int `json:"vm_condition_num,omitempty"` - Filters []map[string]string // 查询中自带查询条件，用于拼接 + Filters []map[string]string `json:"filters,omitempty"` // 查询中自带查询条件，用于拼接 - OffsetInfo OffSetInfo // limit等偏移量配置 + OffsetInfo OffSetInfo `json:"offset_info,omitempty"` // limit等偏移量配置 - SegmentedEnable bool // 是否开启分段查询 + SegmentedEnable bool `json:"segmented_enable,omitempty"` // 是否开启分段查询 // Es 查询扩展 - QueryString string - AllConditions AllConditions + QueryString string `json:"query_string,omitempty"` + AllConditions AllConditions `json:"all_conditions,omitempty"` - HighLight *HighLight + HighLight *HighLight `json:"high_light,omitempty"` - Source []string - From int - Size int + Source []string `json:"source,omitempty"` + From int `json:"from,omitempty"` + Size int `json:"size,omitempty"` - Scroll string - ResultTableOptions ResultTableOptions + Scroll string `json:"scroll,omitempty"` + ResultTableOptions ResultTableOptions `json:"result_table_options,omitempty"` - Orders Orders - NeedAddTime bool + Orders Orders `json:"orders,omitempty"` + NeedAddTime bool `json:"need_add_time,omitempty"` } type HighLight struct { @@ -269,10 +269,13 @@ func ReplaceVmCondition(condition VmCondition, replaceLabels ReplaceLabels) VmCo } // ToJson 通过 tableID 排序，并且返回 json 序列化 -func (qMetric QueryMetric) ToJson(isSort bool) string { +func (qMetric *QueryMetric) ToJson(isSort bool) string { if isSort { - sort.SliceIsSorted(qMetric.QueryList, func(i, j int) bool { - return qMetric.QueryList[i].TableID < qMetric.QueryList[j].TableID + sort.SliceStable(qMetric.QueryList, func(i, j int) bool { + a := qMetric.QueryList[i].TableID + b := qMetric.QueryList[j].TableID + c := a < b + return c }) } diff --git a/pkg/unify-query/mock/handler.go b/pkg/unify-query/mock/handler.go index 65b429234..417cfa9d8 100644 --- a/pkg/unify-query/mock/handler.go +++ b/pkg/unify-query/mock/handler.go @@ -158,16 +158,23 @@ type elasticSearchResultData struct { func mockHandler(ctx context.Context) { httpmock.Activate() - mockVmHandler(ctx) + log.Infof(context.Background(), "mock handler start") + + mockBKBaseHandler(ctx) mockInfluxDBHandler(ctx) - mockBkSQLHandler(ctx) mockElasticSearchHandler(ctx) + + log.Infof(context.Background(), "mock handler end") } const ( - EsUrl = "http://127.0.0.1:93002" - BkSQLUrl = "http://127.0.0.1:92001" - VmUrl = "http://127.0.0.1:12001/bk_data/query_sync" + EsUrlDomain = "http://127.0.0.1:93002" + BkBaseUrlDomain = "http://127.0.0.1:12001" +) + +const ( + EsUrl = EsUrlDomain + BkBaseUrl = BkBaseUrlDomain + "/bk_data/query_sync" ) type BkSQLRequest struct { @@ -179,7 +186,7 @@ type BkSQLRequest struct { } func mockElasticSearchHandler(ctx context.Context) { - bkBaseUrl := "http://127.0.0.1:12001/bk_data/query_sync/es" + bkBaseEsUrl := BkBaseUrl + "/es" searchHandler := func(r *http.Request) (w *http.Response, err error) { body, _ := io.ReadAll(r.Body) @@ -196,45 +203,20 @@ func mockElasticSearchHandler(ctx context.Context) { mappings := `{"es_index":{"mappings":{"properties":{"a":{"type":"keyword"},"b":{"type":"keyword"},"group":{"type":"keyword"},"kibana_stats":{"properties":{"kibana":{"properties":{"name":{"type":"keyword"}}}}},"timestamp":{"type":"log"},"type":{"type":"keyword"},"dtEventTimeStamp":{"type":"date"},"user":{"type":"nested","properties":{"first":{"type":"keyword"},"last":{"type":"keyword"}}}}}}}` mappingResp := httpmock.NewStringResponder(http.StatusOK, mappings) - httpmock.RegisterResponder(http.MethodGet, bkBaseUrl+"/es_index/_mapping/", mappingResp) + httpmock.RegisterResponder(http.MethodGet, bkBaseEsUrl+"/es_index/_mapping/", mappingResp) httpmock.RegisterResponder(http.MethodGet, EsUrl+"/es_index/_mapping/", mappingResp) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/es_index/_search", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search", searchHandler) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/es_index/_search?scroll=5m", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/es_index/_search?scroll=5m", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/es_index/_search?scroll=5m", searchHandler) - httpmock.RegisterResponder(http.MethodPost, bkBaseUrl+"/_search/scroll", searchHandler) + httpmock.RegisterResponder(http.MethodPost, bkBaseEsUrl+"/_search/scroll", searchHandler) httpmock.RegisterResponder(http.MethodPost, EsUrl+"/_search/scroll", searchHandler) - httpmock.RegisterResponder(http.MethodHead, EsUrl, searchHandler) -} - -func mockBkSQLHandler(ctx context.Context) { - httpmock.RegisterResponder(http.MethodPost, BkSQLUrl, func(r *http.Request) (w *http.Response, err error) { - var ( - request BkSQLRequest - ) - err = json.NewDecoder(r.Body).Decode(&request) - if err != nil { - return - } - - d, ok := BkSQL.Get(request.Sql) - if !ok { - err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) - log.Errorf(ctx, err.Error()) - return - } - - switch t := d.(type) { - case string: - w = httpmock.NewStringResponse(http.StatusOK, t) - default: - w, err = httpmock.NewJsonResponse(http.StatusOK, d) - } - return + httpmock.RegisterResponder(http.MethodHead, EsUrl, func(request *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(http.StatusOK, ""), nil }) } @@ -268,8 +250,8 @@ func mockInfluxDBHandler(ctx context.Context) { }) } -func mockVmHandler(ctx context.Context) { - httpmock.RegisterResponder(http.MethodPost, VmUrl, func(r *http.Request) (w *http.Response, err error) { +func mockBKBaseHandler(ctx context.Context) { + httpmock.RegisterResponder(http.MethodPost, BkBaseUrl, func(r *http.Request) (w *http.Response, err error) { var ( request VmRequest params VmParams @@ -279,6 +261,23 @@ func mockVmHandler(ctx context.Context) { return } + if request.PreferStorage != "vm" { + d, ok := BkSQL.Get(request.Sql) + if !ok { + err = fmt.Errorf(`bksql mock data is empty in "%s"`, request.Sql) + log.Errorf(ctx, err.Error()) + return + } + switch t := d.(type) { + case string: + w = httpmock.NewStringResponse(http.StatusOK, t) + default: + w, err = httpmock.NewJsonResponse(http.StatusOK, d) + } + + return + } + err = json.Unmarshal([]byte(request.Sql), ¶ms) if err != nil { return diff --git a/pkg/unify-query/query/structured/method.go b/pkg/unify-query/query/structured/method.go index 611221c79..9b4510e77 100644 --- a/pkg/unify-query/query/structured/method.go +++ b/pkg/unify-query/query/structured/method.go @@ -48,7 +48,7 @@ func (a AggregateMethodList) ToQry(timezone string) (metadata.Aggregates, error) for _, aggr := range a { agg := metadata.Aggregate{ Name: aggr.Method, - Dimensions: aggr.Dimensions, + Dimensions: append([]string{}, aggr.Dimensions...), Without: aggr.Without, Args: aggr.VArgsList, } diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index ca5a55782..afdef2fe7 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -465,7 +465,7 @@ func (q *Query) Aggregates() (aggs metadata.Aggregates, err error) { agg := metadata.Aggregate{ Name: name, Field: am.Field, - Dimensions: am.Dimensions, + Dimensions: append([]string{}, am.Dimensions...), Without: am.Without, Window: time.Duration(window), TimeZone: q.Timezone, diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index e8bca55e8..1975356bb 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -11,7 +11,6 @@ package structured import ( "context" - "encoding/json" "testing" "time" @@ -272,10 +271,7 @@ func TestQueryToMetric(t *testing.T) { metric, err := c.query.ToQueryMetric(ctx, spaceUID) assert.Nil(t, err) - a, _ := json.Marshal(c.metric) - b, _ := json.Marshal(metric) - - assert.JSONEq(t, string(a), string(b)) + assert.Equal(t, c.metric.ToJson(true), metric.ToJson(true)) }) } } @@ -1141,7 +1137,6 @@ func TestQueryTs_ToQueryReference(t *testing.T) { { DataSource: BkLog, Timezone: "UTC", - SourceType: "bkdata", TableID: "result_table.es", DataLabel: "es", DB: "es_index", diff --git a/pkg/unify-query/query/structured/space.go b/pkg/unify-query/query/structured/space.go index 4cb3274e3..377a1c5f4 100644 --- a/pkg/unify-query/query/structured/space.go +++ b/pkg/unify-query/query/structured/space.go @@ -87,7 +87,7 @@ func (s *SpaceFilter) getTsDBWithResultTableDetail(t query.TsDBV2, d *routerInfl t.NeedAddTime = d.Options.NeedAddTime t.SourceType = d.SourceType - sort.SliceIsSorted(d.StorageClusterRecords, func(i, j int) bool { + sort.SliceStable(d.StorageClusterRecords, func(i, j int) bool { return d.StorageClusterRecords[i].EnableTime > d.StorageClusterRecords[j].EnableTime }) diff --git a/pkg/unify-query/query/structured/space_test.go b/pkg/unify-query/query/structured/space_test.go index 929662170..3911bbc7a 100644 --- a/pkg/unify-query/query/structured/space_test.go +++ b/pkg/unify-query/query/structured/space_test.go @@ -43,17 +43,17 @@ func TestSpaceFilter_NewTsDBs(t *testing.T) { "test_1": { fieldName: "kube_node_info", tableID: "", - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"influxdb"}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kube_node_info","expand_metric_names":["kube_node_info"],"time_field":{},"need_add_time":false,"storage_type":"influxdb"}]`, }, "test_2_regex": { fieldName: "kubelet_.+", isRegexp: true, - expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"influxdb"},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"victoria_metrics"}]`, + expected: `[{"table_id":"result_table.influxdb","field":["kube_pod_info","kube_node_info","kube_node_status_condition","kubelet_cluster_request_total"],"measurement_type":"bk_split_measurement","data_label":"influxdb","storage_id":"2","cluster_name":"default","db":"result_table","measurement":"influxdb","metric_name":"kubelet_.+","expand_metric_names":["kubelet_cluster_request_total"],"time_field":{},"need_add_time":false,"storage_type":"influxdb"},{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"kubelet_info","storage_id":"2","db":"other","measurement":"kubelet_info","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"kubelet_.+","expand_metric_names":["kubelet_info"],"time_field":{},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, "test_3_regex": { fieldName: "container_.+", isRegexp: true, - expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{"Name":"","Type":"","Unit":""},"need_add_time":false,"storage_type":"victoria_metrics"}]`, + expected: `[{"table_id":"result_table.vm","field":["container_cpu_usage_seconds_total","kube_pod_info","node_with_pod_relation","node_with_system_relation","deployment_with_replicaset_relation","pod_with_replicaset_relation","apm_service_instance_with_pod_relation","apm_service_instance_with_system_relation","kubelet_info"],"measurement_type":"bk_split_measurement","data_label":"vm","storage_id":"2","vm_rt":"2_bcs_prom_computation_result_table","metric_name":"container_.+","expand_metric_names":["container_cpu_usage_seconds_total"],"time_field":{},"need_add_time":false,"storage_type":"victoria_metrics"}]`, }, } @@ -84,8 +84,8 @@ func TestSpaceFilter_NewTsDBs(t *testing.T) { } func toJson(q []*query.TsDBV2) string { - sort.SliceIsSorted(q, func(i, j int) bool { - return q[i].TableID < q[i].TableID + sort.SliceStable(q, func(i, j int) bool { + return q[i].TableID < q[j].TableID }) s, _ := json.Marshal(q) diff --git a/pkg/unify-query/service/http/handler.go b/pkg/unify-query/service/http/handler.go index e1602dd25..92bc70c26 100644 --- a/pkg/unify-query/service/http/handler.go +++ b/pkg/unify-query/service/http/handler.go @@ -21,7 +21,6 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/trace" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/elasticsearch" ) // HandlerPromQLToStruct @@ -199,13 +198,12 @@ func HandlerQueryExemplar(c *gin.Context) { // @Router /query/raw [post] func HandlerQueryRaw(c *gin.Context) { var ( - ctx = c.Request.Context() - resp = &response{c: c} - user = metadata.GetUser(ctx) - err error - span *trace.Span - listData ListData - ignoreDimensions = []string{elasticsearch.KeyAddress} + ctx = c.Request.Context() + resp = &response{c: c} + user = metadata.GetUser(ctx) + err error + span *trace.Span + listData ListData ) ctx, span = trace.NewSpan(ctx, "handler-query-raw") @@ -241,7 +239,7 @@ func HandlerQueryRaw(c *gin.Context) { listData.TraceID = span.TraceID() - total, list, resultTableOptions, err := queryRawWithInstance(ctx, queryTs) + listData.Total, listData.List, listData.ResultTableOptions, err = queryRawWithInstance(ctx, queryTs) if err != nil { listData.Status = &metadata.Status{ Code: metadata.QueryRawError, @@ -249,19 +247,6 @@ func HandlerQueryRaw(c *gin.Context) { } return } - listData.Total = total - listData.List = make([]map[string]any, 0, len(list)) - for _, item := range list { - if item == nil { - continue - } - - for _, ignoreDimension := range ignoreDimensions { - delete(item, ignoreDimension) - } - listData.List = append(listData.List, item) - } - listData.ResultTableOptions = resultTableOptions resp.success(ctx, listData) } diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 02d86fced..b1d46cbe3 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -158,6 +158,8 @@ func queryExemplar(ctx context.Context, query *structured.QueryTs) (interface{}, } func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (total int64, list []map[string]any, resultTableOptions metadata.ResultTableOptions, err error) { + ignoreDimensions := []string{elasticsearch.KeyAddress} + ctx, span := trace.NewSpan(ctx, "query-raw-with-instance") defer span.End(&err) @@ -250,14 +252,15 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot go func() { defer receiveWg.Done() + var datas []map[string]any for d := range dataCh { - list = append(list, d) + datas = append(datas, d) } span.Set("query-list-num", len(queryList)) if len(queryList) > 1 { - queryTs.OrderBy.Orders().SortSliceList(list) + queryTs.OrderBy.Orders().SortSliceList(datas) span.Set("query-scroll", queryTs.Scroll) span.Set("query-result-table", queryTs.ResultTableOptions) @@ -266,19 +269,19 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot if queryTs.Scroll == "" && queryTs.ResultTableOptions.IsCrop() { // 判定是否启用 multi from 特性 span.Set("query-multi-from", queryTs.IsMultiFrom) - span.Set("list-length", len(list)) + span.Set("datas-length", len(datas)) span.Set("query-ts-from", queryTs.From) span.Set("query-ts-limit", queryTs.Limit) - if len(list) > queryTs.Limit { + if len(datas) > queryTs.Limit { if queryTs.IsMultiFrom { resultTableOptions = queryTs.ResultTableOptions if resultTableOptions == nil { resultTableOptions = make(metadata.ResultTableOptions) } - list = list[0:queryTs.Limit] - for _, l := range list { + datas = datas[0:queryTs.Limit] + for _, l := range datas { tableID := l[elasticsearch.KeyTableID].(string) address := l[elasticsearch.KeyAddress].(string) @@ -290,10 +293,22 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } } } else { - list = list[queryTs.From : queryTs.From+queryTs.Limit] + datas = datas[queryTs.From : queryTs.From+queryTs.Limit] } } } + + } + + for _, item := range datas { + if item == nil { + continue + } + + for _, ignoreDimension := range ignoreDimensions { + delete(item, ignoreDimension) + } + list = append(list, item) } }() diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index f83ca3903..cac4729ae 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -17,8 +17,10 @@ import ( "time" "github.com/prometheus/prometheus/promql/parser" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/bkapi" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/consul" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/featureFlag" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/influxdb" @@ -35,9 +37,134 @@ import ( "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/redis" ) +func TestQueryTsWithDoris(t *testing.T) { + ctx := metadata.InitHashID(context.Background()) + + //viper.Set(bkapi.BkAPIAddressConfigPath, mock.BkSQLUrlDomain) + + spaceUid := influxdb.SpaceUid + tableID := influxdb.ResultTableDoris + + mock.Init() + promql.MockEngine() + influxdb.MockSpaceRouter(ctx) + + defaultStart := time.UnixMilli(1744662513046) + defaultEnd := time.UnixMilli(1744684113046) + + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + // 查询 1 条原始数据，按照字段正向排序 + "SELECT *, `gseIndex` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662180000 AND `dtEventTimeStamp` <= 1744684113000 AND `thedate` = '20250415' ORDER BY `_value_` ASC LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":12,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250415,"dteventtimestamp":1744662180000,"dteventtime":null,"localtime":null,"__shard_key__":29077703953,"__ext":"{\"container_id\":\"375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2\",\"container_image\":\"sha256:3a0506f06f1467e93c3a582203aac1a7501e77091572ec9612ddeee4a4dbbdb8\",\"container_name\":\"unify-query\",\"io_kubernetes_pod\":\"bk-datalink-unify-query-6df8bcc4c9-rk4sc\",\"io_kubernetes_pod_ip\":\"127.0.0.1\",\"io_kubernetes_pod_namespace\":\"blueking\",\"io_kubernetes_pod_uid\":\"558c5b17-b221-47e1-aa66-036cc9b43e2a\",\"io_kubernetes_workload_name\":\"bk-datalink-unify-query-6df8bcc4c9\",\"io_kubernetes_workload_type\":\"ReplicaSet\"}","cloudid":0.0,"file":"http/handler.go:320","gseindex":2450131.0,"iterationindex":19.0,"level":"info","log":"2025-04-14T20:22:59.982Z\tinfo\thttp/handler.go:320\t[5108397435e997364f8dc1251533e65e] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2-json.log","report_time":"2025-04-14T20:22:59.982Z","serverip":"30.189.37.111","time":"1744662180000","trace_id":"5108397435e997364f8dc1251533e65e","_value_":2450131.0,"_timestamp_":1744662180000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":182,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":56,"match_query_routing_rule":0,"check_permission":13,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","__shard_key__","__ext","cloudid","file","gseindex","iterationindex","level","log","message","path","report_time","serverip","time","trace_id","_value_","_timestamp_"],"total_record_size":8856,"timetaken":0.255,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"long","field_name":"__c4","field_alias":"__shard_key__","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"__ext","field_index":5},{"field_type":"double","field_name":"__c6","field_alias":"cloudid","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"file","field_index":7},{"field_type":"double","field_name":"__c8","field_alias":"gseindex","field_index":8},{"field_type":"double","field_name":"__c9","field_alias":"iterationindex","field_index":9},{"field_type":"string","field_name":"__c10","field_alias":"level","field_index":10},{"field_type":"string","field_name":"__c11","field_alias":"log","field_index":11},{"field_type":"string","field_name":"__c12","field_alias":"message","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"path","field_index":13},{"field_type":"string","field_name":"__c14","field_alias":"report_time","field_index":14},{"field_type":"string","field_name":"__c15","field_alias":"serverip","field_index":15},{"field_type":"string","field_name":"__c16","field_alias":"time","field_index":16},{"field_type":"string","field_name":"__c17","field_alias":"trace_id","field_index":17},{"field_type":"double","field_name":"__c18","field_alias":"_value_","field_index":18},{"field_type":"long","field_name":"__c19","field_alias":"_timestamp_","field_index":19}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + + // 根据维度 __ext.container_name 进行 count 聚合，同时用值正向排序 + "SELECT CAST(__ext[\"container_name\"] AS STRING) AS `__ext__bk_46__container_name`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 30000 AS INT) * 30000 AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662509999 AND `dtEventTimeStamp` <= 1744684142999 AND `thedate` = '20250415' GROUP BY CAST(__ext[\"container_name\"] AS STRING), _timestamp_ ORDER BY `_timestamp_` ASC, `_value_` ASC LIMIT 2000005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":722,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":6,"bkbase_apigw_api":28},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"__ext__bk_46__container_name":"unify-query","_value_":3684,"_timestamp_":1744662510000},{"__ext__bk_46__container_name":"unify-query","_value_":4012,"_timestamp_":1744662540000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744662570000},{"__ext__bk_46__container_name":"unify-query","_value_":17092,"_timestamp_":1744662600000},{"__ext__bk_46__container_name":"unify-query","_value_":12881,"_timestamp_":1744662630000},{"__ext__bk_46__container_name":"unify-query","_value_":5902,"_timestamp_":1744662660000},{"__ext__bk_46__container_name":"unify-query","_value_":10443,"_timestamp_":1744662690000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744662720000},{"__ext__bk_46__container_name":"unify-query","_value_":3357,"_timestamp_":1744662750000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744662780000},{"__ext__bk_46__container_name":"unify-query","_value_":3683,"_timestamp_":1744662810000},{"__ext__bk_46__container_name":"unify-query","_value_":4353,"_timestamp_":1744662840000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744662870000},{"__ext__bk_46__container_name":"unify-query","_value_":4251,"_timestamp_":1744662900000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744662930000},{"__ext__bk_46__container_name":"unify-query","_value_":4036,"_timestamp_":1744662960000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744662990000},{"__ext__bk_46__container_name":"unify-query","_value_":4351,"_timestamp_":1744663020000},{"__ext__bk_46__container_name":"unify-query","_value_":3651,"_timestamp_":1744663050000},{"__ext__bk_46__container_name":"unify-query","_value_":4096,"_timestamp_":1744663080000},{"__ext__bk_46__container_name":"unify-query","_value_":3618,"_timestamp_":1744663110000},{"__ext__bk_46__container_name":"unify-query","_value_":4100,"_timestamp_":1744663140000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744663170000},{"__ext__bk_46__container_name":"unify-query","_value_":6044,"_timestamp_":1744663200000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744663230000},{"__ext__bk_46__container_name":"unify-query","_value_":4461,"_timestamp_":1744663260000},{"__ext__bk_46__container_name":"unify-query","_value_":3783,"_timestamp_":1744663290000},{"__ext__bk_46__container_name":"unify-query","_value_":4559,"_timestamp_":1744663320000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744663350000},{"__ext__bk_46__container_name":"unify-query","_value_":3869,"_timestamp_":1744663380000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744663410000},{"__ext__bk_46__container_name":"unify-query","_value_":4473,"_timestamp_":1744663440000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744663470000},{"__ext__bk_46__container_name":"unify-query","_value_":4923,"_timestamp_":1744663500000},{"__ext__bk_46__container_name":"unify-query","_value_":3379,"_timestamp_":1744663530000},{"__ext__bk_46__container_name":"unify-query","_value_":4489,"_timestamp_":1744663560000},{"__ext__bk_46__container_name":"unify-query","_value_":3411,"_timestamp_":1744663590000},{"__ext__bk_46__container_name":"unify-query","_value_":4374,"_timestamp_":1744663620000},{"__ext__bk_46__container_name":"unify-query","_value_":3370,"_timestamp_":1744663650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744663680000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744663710000},{"__ext__bk_46__container_name":"unify-query","_value_":4318,"_timestamp_":1744663740000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744663770000},{"__ext__bk_46__container_name":"unify-query","_value_":4334,"_timestamp_":1744663800000},{"__ext__bk_46__container_name":"unify-query","_value_":3767,"_timestamp_":1744663830000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744663860000},{"__ext__bk_46__container_name":"unify-query","_value_":3703,"_timestamp_":1744663890000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744663920000},{"__ext__bk_46__container_name":"unify-query","_value_":3667,"_timestamp_":1744663950000},{"__ext__bk_46__container_name":"unify-query","_value_":3998,"_timestamp_":1744663980000},{"__ext__bk_46__container_name":"unify-query","_value_":3579,"_timestamp_":1744664010000},{"__ext__bk_46__container_name":"unify-query","_value_":4156,"_timestamp_":1744664040000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744664070000},{"__ext__bk_46__container_name":"unify-query","_value_":4344,"_timestamp_":1744664100000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744664130000},{"__ext__bk_46__container_name":"unify-query","_value_":4161,"_timestamp_":1744664160000},{"__ext__bk_46__container_name":"unify-query","_value_":3484,"_timestamp_":1744664190000},{"__ext__bk_46__container_name":"unify-query","_value_":4273,"_timestamp_":1744664220000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744664250000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744664280000},{"__ext__bk_46__container_name":"unify-query","_value_":3619,"_timestamp_":1744664310000},{"__ext__bk_46__container_name":"unify-query","_value_":4013,"_timestamp_":1744664340000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744664370000},{"__ext__bk_46__container_name":"unify-query","_value_":18144,"_timestamp_":1744664400000},{"__ext__bk_46__container_name":"unify-query","_value_":13615,"_timestamp_":1744664430000},{"__ext__bk_46__container_name":"unify-query","_value_":3178,"_timestamp_":1744664460000},{"__ext__bk_46__container_name":"unify-query","_value_":13044,"_timestamp_":1744664490000},{"__ext__bk_46__container_name":"unify-query","_value_":4767,"_timestamp_":1744664520000},{"__ext__bk_46__container_name":"unify-query","_value_":3528,"_timestamp_":1744664550000},{"__ext__bk_46__container_name":"unify-query","_value_":4316,"_timestamp_":1744664580000},{"__ext__bk_46__container_name":"unify-query","_value_":3317,"_timestamp_":1744664610000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744664640000},{"__ext__bk_46__container_name":"unify-query","_value_":3599,"_timestamp_":1744664670000},{"__ext__bk_46__container_name":"unify-query","_value_":4149,"_timestamp_":1744664700000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744664730000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744664760000},{"__ext__bk_46__container_name":"unify-query","_value_":3384,"_timestamp_":1744664790000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744664820000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744664850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744664880000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744664910000},{"__ext__bk_46__container_name":"unify-query","_value_":4244,"_timestamp_":1744664940000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744664970000},{"__ext__bk_46__container_name":"unify-query","_value_":4305,"_timestamp_":1744665000000},{"__ext__bk_46__container_name":"unify-query","_value_":3771,"_timestamp_":1744665030000},{"__ext__bk_46__container_name":"unify-query","_value_":4485,"_timestamp_":1744665060000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744665090000},{"__ext__bk_46__container_name":"unify-query","_value_":4423,"_timestamp_":1744665120000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665150000},{"__ext__bk_46__container_name":"unify-query","_value_":4125,"_timestamp_":1744665180000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744665210000},{"__ext__bk_46__container_name":"unify-query","_value_":4252,"_timestamp_":1744665240000},{"__ext__bk_46__container_name":"unify-query","_value_":3427,"_timestamp_":1744665270000},{"__ext__bk_46__container_name":"unify-query","_value_":5089,"_timestamp_":1744665300000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744665330000},{"__ext__bk_46__container_name":"unify-query","_value_":4349,"_timestamp_":1744665360000},{"__ext__bk_46__container_name":"unify-query","_value_":3188,"_timestamp_":1744665390000},{"__ext__bk_46__container_name":"unify-query","_value_":4556,"_timestamp_":1744665420000},{"__ext__bk_46__container_name":"unify-query","_value_":3372,"_timestamp_":1744665450000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744665480000},{"__ext__bk_46__container_name":"unify-query","_value_":3445,"_timestamp_":1744665510000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744665540000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744665570000},{"__ext__bk_46__container_name":"unify-query","_value_":6235,"_timestamp_":1744665600000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744665630000},{"__ext__bk_46__container_name":"unify-query","_value_":4577,"_timestamp_":1744665660000},{"__ext__bk_46__container_name":"unify-query","_value_":3719,"_timestamp_":1744665690000},{"__ext__bk_46__container_name":"unify-query","_value_":4548,"_timestamp_":1744665720000},{"__ext__bk_46__container_name":"unify-query","_value_":3420,"_timestamp_":1744665750000},{"__ext__bk_46__container_name":"unify-query","_value_":4246,"_timestamp_":1744665780000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744665810000},{"__ext__bk_46__container_name":"unify-query","_value_":4332,"_timestamp_":1744665840000},{"__ext__bk_46__container_name":"unify-query","_value_":3422,"_timestamp_":1744665870000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744665900000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665930000},{"__ext__bk_46__container_name":"unify-query","_value_":4119,"_timestamp_":1744665960000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744665990000},{"__ext__bk_46__container_name":"unify-query","_value_":4144,"_timestamp_":1744666020000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744666050000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744666080000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744666110000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744666140000},{"__ext__bk_46__container_name":"unify-query","_value_":3315,"_timestamp_":1744666170000},{"__ext__bk_46__container_name":"unify-query","_value_":16721,"_timestamp_":1744666200000},{"__ext__bk_46__container_name":"unify-query","_value_":13631,"_timestamp_":1744666230000},{"__ext__bk_46__container_name":"unify-query","_value_":2982,"_timestamp_":1744666260000},{"__ext__bk_46__container_name":"unify-query","_value_":11858,"_timestamp_":1744666290000},{"__ext__bk_46__container_name":"unify-query","_value_":5515,"_timestamp_":1744666320000},{"__ext__bk_46__container_name":"unify-query","_value_":2869,"_timestamp_":1744666350000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744666380000},{"__ext__bk_46__container_name":"unify-query","_value_":3603,"_timestamp_":1744666410000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744666440000},{"__ext__bk_46__container_name":"unify-query","_value_":3264,"_timestamp_":1744666470000},{"__ext__bk_46__container_name":"unify-query","_value_":4377,"_timestamp_":1744666500000},{"__ext__bk_46__container_name":"unify-query","_value_":3443,"_timestamp_":1744666530000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744666560000},{"__ext__bk_46__container_name":"unify-query","_value_":3459,"_timestamp_":1744666590000},{"__ext__bk_46__container_name":"unify-query","_value_":4342,"_timestamp_":1744666620000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744666650000},{"__ext__bk_46__container_name":"unify-query","_value_":4052,"_timestamp_":1744666680000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744666710000},{"__ext__bk_46__container_name":"unify-query","_value_":4128,"_timestamp_":1744666740000},{"__ext__bk_46__container_name":"unify-query","_value_":3499,"_timestamp_":1744666770000},{"__ext__bk_46__container_name":"unify-query","_value_":6209,"_timestamp_":1744666800000},{"__ext__bk_46__container_name":"unify-query","_value_":3575,"_timestamp_":1744666830000},{"__ext__bk_46__container_name":"unify-query","_value_":4543,"_timestamp_":1744666860000},{"__ext__bk_46__container_name":"unify-query","_value_":3604,"_timestamp_":1744666890000},{"__ext__bk_46__container_name":"unify-query","_value_":4579,"_timestamp_":1744666920000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744666950000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744666980000},{"__ext__bk_46__container_name":"unify-query","_value_":3416,"_timestamp_":1744667010000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667040000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744667070000},{"__ext__bk_46__container_name":"unify-query","_value_":5054,"_timestamp_":1744667100000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744667130000},{"__ext__bk_46__container_name":"unify-query","_value_":4313,"_timestamp_":1744667160000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744667190000},{"__ext__bk_46__container_name":"unify-query","_value_":4118,"_timestamp_":1744667220000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744667250000},{"__ext__bk_46__container_name":"unify-query","_value_":4280,"_timestamp_":1744667280000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744667310000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744667340000},{"__ext__bk_46__container_name":"unify-query","_value_":3807,"_timestamp_":1744667370000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744667400000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744667430000},{"__ext__bk_46__container_name":"unify-query","_value_":4630,"_timestamp_":1744667460000},{"__ext__bk_46__container_name":"unify-query","_value_":3845,"_timestamp_":1744667490000},{"__ext__bk_46__container_name":"unify-query","_value_":4361,"_timestamp_":1744667520000},{"__ext__bk_46__container_name":"unify-query","_value_":3572,"_timestamp_":1744667550000},{"__ext__bk_46__container_name":"unify-query","_value_":4095,"_timestamp_":1744667580000},{"__ext__bk_46__container_name":"unify-query","_value_":3535,"_timestamp_":1744667610000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744667640000},{"__ext__bk_46__container_name":"unify-query","_value_":3390,"_timestamp_":1744667670000},{"__ext__bk_46__container_name":"unify-query","_value_":4262,"_timestamp_":1744667700000},{"__ext__bk_46__container_name":"unify-query","_value_":3398,"_timestamp_":1744667730000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667760000},{"__ext__bk_46__container_name":"unify-query","_value_":3429,"_timestamp_":1744667790000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744667820000},{"__ext__bk_46__container_name":"unify-query","_value_":3482,"_timestamp_":1744667850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744667880000},{"__ext__bk_46__container_name":"unify-query","_value_":3612,"_timestamp_":1744667910000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744667940000},{"__ext__bk_46__container_name":"unify-query","_value_":3423,"_timestamp_":1744667970000},{"__ext__bk_46__container_name":"unify-query","_value_":18203,"_timestamp_":1744668000000},{"__ext__bk_46__container_name":"unify-query","_value_":13685,"_timestamp_":1744668030000},{"__ext__bk_46__container_name":"unify-query","_value_":3281,"_timestamp_":1744668060000},{"__ext__bk_46__container_name":"unify-query","_value_":12556,"_timestamp_":1744668090000},{"__ext__bk_46__container_name":"unify-query","_value_":4893,"_timestamp_":1744668120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744668150000},{"__ext__bk_46__container_name":"unify-query","_value_":4336,"_timestamp_":1744668180000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744668210000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744668240000},{"__ext__bk_46__container_name":"unify-query","_value_":3669,"_timestamp_":1744668270000},{"__ext__bk_46__container_name":"unify-query","_value_":3997,"_timestamp_":1744668300000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744668330000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744668360000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744668390000},{"__ext__bk_46__container_name":"unify-query","_value_":3877,"_timestamp_":1744668420000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744668450000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744668480000},{"__ext__bk_46__container_name":"unify-query","_value_":3469,"_timestamp_":1744668510000},{"__ext__bk_46__container_name":"unify-query","_value_":4243,"_timestamp_":1744668540000},{"__ext__bk_46__container_name":"unify-query","_value_":3304,"_timestamp_":1744668570000},{"__ext__bk_46__container_name":"unify-query","_value_":4690,"_timestamp_":1744668600000},{"__ext__bk_46__container_name":"unify-query","_value_":3717,"_timestamp_":1744668630000},{"__ext__bk_46__container_name":"unify-query","_value_":4618,"_timestamp_":1744668660000},{"__ext__bk_46__container_name":"unify-query","_value_":3732,"_timestamp_":1744668690000},{"__ext__bk_46__container_name":"unify-query","_value_":4477,"_timestamp_":1744668720000},{"__ext__bk_46__container_name":"unify-query","_value_":3615,"_timestamp_":1744668750000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744668780000},{"__ext__bk_46__container_name":"unify-query","_value_":3367,"_timestamp_":1744668810000},{"__ext__bk_46__container_name":"unify-query","_value_":4193,"_timestamp_":1744668840000},{"__ext__bk_46__container_name":"unify-query","_value_":3592,"_timestamp_":1744668870000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744668900000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744668930000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744668960000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744668990000},{"__ext__bk_46__container_name":"unify-query","_value_":4375,"_timestamp_":1744669020000},{"__ext__bk_46__container_name":"unify-query","_value_":3386,"_timestamp_":1744669050000},{"__ext__bk_46__container_name":"unify-query","_value_":4281,"_timestamp_":1744669080000},{"__ext__bk_46__container_name":"unify-query","_value_":3410,"_timestamp_":1744669110000},{"__ext__bk_46__container_name":"unify-query","_value_":4545,"_timestamp_":1744669140000},{"__ext__bk_46__container_name":"unify-query","_value_":3724,"_timestamp_":1744669170000},{"__ext__bk_46__container_name":"unify-query","_value_":5903,"_timestamp_":1744669200000},{"__ext__bk_46__container_name":"unify-query","_value_":3672,"_timestamp_":1744669230000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744669260000},{"__ext__bk_46__container_name":"unify-query","_value_":3792,"_timestamp_":1744669290000},{"__ext__bk_46__container_name":"unify-query","_value_":4422,"_timestamp_":1744669320000},{"__ext__bk_46__container_name":"unify-query","_value_":3718,"_timestamp_":1744669350000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744669380000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744669410000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744669440000},{"__ext__bk_46__container_name":"unify-query","_value_":3542,"_timestamp_":1744669470000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744669500000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669530000},{"__ext__bk_46__container_name":"unify-query","_value_":4354,"_timestamp_":1744669560000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669590000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744669620000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744669650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744669680000},{"__ext__bk_46__container_name":"unify-query","_value_":3512,"_timestamp_":1744669710000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744669740000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744669770000},{"__ext__bk_46__container_name":"unify-query","_value_":12171,"_timestamp_":1744669800000},{"__ext__bk_46__container_name":"unify-query","_value_":18129,"_timestamp_":1744669830000},{"__ext__bk_46__container_name":"unify-query","_value_":7142,"_timestamp_":1744669860000},{"__ext__bk_46__container_name":"unify-query","_value_":9153,"_timestamp_":1744669890000},{"__ext__bk_46__container_name":"unify-query","_value_":4566,"_timestamp_":1744669920000},{"__ext__bk_46__container_name":"unify-query","_value_":3225,"_timestamp_":1744669950000},{"__ext__bk_46__container_name":"unify-query","_value_":4378,"_timestamp_":1744669980000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744670010000},{"__ext__bk_46__container_name":"unify-query","_value_":4266,"_timestamp_":1744670040000},{"__ext__bk_46__container_name":"unify-query","_value_":3645,"_timestamp_":1744670070000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744670100000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744670130000},{"__ext__bk_46__container_name":"unify-query","_value_":4333,"_timestamp_":1744670160000},{"__ext__bk_46__container_name":"unify-query","_value_":3489,"_timestamp_":1744670190000},{"__ext__bk_46__container_name":"unify-query","_value_":4303,"_timestamp_":1744670220000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744670250000},{"__ext__bk_46__container_name":"unify-query","_value_":4121,"_timestamp_":1744670280000},{"__ext__bk_46__container_name":"unify-query","_value_":3374,"_timestamp_":1744670310000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744670340000},{"__ext__bk_46__container_name":"unify-query","_value_":3242,"_timestamp_":1744670370000},{"__ext__bk_46__container_name":"unify-query","_value_":6416,"_timestamp_":1744670400000},{"__ext__bk_46__container_name":"unify-query","_value_":3697,"_timestamp_":1744670430000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744670460000},{"__ext__bk_46__container_name":"unify-query","_value_":3749,"_timestamp_":1744670490000},{"__ext__bk_46__container_name":"unify-query","_value_":4587,"_timestamp_":1744670520000},{"__ext__bk_46__container_name":"unify-query","_value_":3538,"_timestamp_":1744670550000},{"__ext__bk_46__container_name":"unify-query","_value_":4221,"_timestamp_":1744670580000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744670610000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744670640000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744670670000},{"__ext__bk_46__container_name":"unify-query","_value_":4848,"_timestamp_":1744670700000},{"__ext__bk_46__container_name":"unify-query","_value_":3551,"_timestamp_":1744670730000},{"__ext__bk_46__container_name":"unify-query","_value_":4068,"_timestamp_":1744670760000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744670790000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744670820000},{"__ext__bk_46__container_name":"unify-query","_value_":3635,"_timestamp_":1744670850000},{"__ext__bk_46__container_name":"unify-query","_value_":4256,"_timestamp_":1744670880000},{"__ext__bk_46__container_name":"unify-query","_value_":3690,"_timestamp_":1744670910000},{"__ext__bk_46__container_name":"unify-query","_value_":4155,"_timestamp_":1744670940000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744670970000},{"__ext__bk_46__container_name":"unify-query","_value_":4661,"_timestamp_":1744671000000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744671030000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744671060000},{"__ext__bk_46__container_name":"unify-query","_value_":3643,"_timestamp_":1744671090000},{"__ext__bk_46__container_name":"unify-query","_value_":4755,"_timestamp_":1744671120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744671150000},{"__ext__bk_46__container_name":"unify-query","_value_":4284,"_timestamp_":1744671180000},{"__ext__bk_46__container_name":"unify-query","_value_":3258,"_timestamp_":1744671210000},{"__ext__bk_46__container_name":"unify-query","_value_":4453,"_timestamp_":1744671240000},{"__ext__bk_46__container_name":"unify-query","_value_":3431,"_timestamp_":1744671270000},{"__ext__bk_46__container_name":"unify-query","_value_":4231,"_timestamp_":1744671300000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744671330000},{"__ext__bk_46__container_name":"unify-query","_value_":3907,"_timestamp_":1744671360000},{"__ext__bk_46__container_name":"unify-query","_value_":3524,"_timestamp_":1744671390000},{"__ext__bk_46__container_name":"unify-query","_value_":4438,"_timestamp_":1744671420000},{"__ext__bk_46__container_name":"unify-query","_value_":3547,"_timestamp_":1744671450000},{"__ext__bk_46__container_name":"unify-query","_value_":4033,"_timestamp_":1744671480000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744671510000},{"__ext__bk_46__container_name":"unify-query","_value_":4162,"_timestamp_":1744671540000},{"__ext__bk_46__container_name":"unify-query","_value_":3588,"_timestamp_":1744671570000},{"__ext__bk_46__container_name":"unify-query","_value_":16444,"_timestamp_":1744671600000},{"__ext__bk_46__container_name":"unify-query","_value_":15396,"_timestamp_":1744671630000},{"__ext__bk_46__container_name":"unify-query","_value_":3024,"_timestamp_":1744671660000},{"__ext__bk_46__container_name":"unify-query","_value_":12656,"_timestamp_":1744671690000},{"__ext__bk_46__container_name":"unify-query","_value_":4733,"_timestamp_":1744671720000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744671750000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744671780000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744671810000},{"__ext__bk_46__container_name":"unify-query","_value_":4487,"_timestamp_":1744671840000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744671870000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744671900000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744671930000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744671960000},{"__ext__bk_46__container_name":"unify-query","_value_":3472,"_timestamp_":1744671990000},{"__ext__bk_46__container_name":"unify-query","_value_":4309,"_timestamp_":1744672020000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744672050000},{"__ext__bk_46__container_name":"unify-query","_value_":4191,"_timestamp_":1744672080000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744672110000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744672140000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744672170000},{"__ext__bk_46__container_name":"unify-query","_value_":4445,"_timestamp_":1744672200000},{"__ext__bk_46__container_name":"unify-query","_value_":3822,"_timestamp_":1744672230000},{"__ext__bk_46__container_name":"unify-query","_value_":4346,"_timestamp_":1744672260000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744672290000},{"__ext__bk_46__container_name":"unify-query","_value_":4615,"_timestamp_":1744672320000},{"__ext__bk_46__container_name":"unify-query","_value_":3591,"_timestamp_":1744672350000},{"__ext__bk_46__container_name":"unify-query","_value_":4056,"_timestamp_":1744672380000},{"__ext__bk_46__container_name":"unify-query","_value_":3544,"_timestamp_":1744672410000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744672440000},{"__ext__bk_46__container_name":"unify-query","_value_":3647,"_timestamp_":1744672470000},{"__ext__bk_46__container_name":"unify-query","_value_":4887,"_timestamp_":1744672500000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744672530000},{"__ext__bk_46__container_name":"unify-query","_value_":4302,"_timestamp_":1744672560000},{"__ext__bk_46__container_name":"unify-query","_value_":3425,"_timestamp_":1744672590000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744672620000},{"__ext__bk_46__container_name":"unify-query","_value_":3532,"_timestamp_":1744672650000},{"__ext__bk_46__container_name":"unify-query","_value_":4282,"_timestamp_":1744672680000},{"__ext__bk_46__container_name":"unify-query","_value_":3571,"_timestamp_":1744672710000},{"__ext__bk_46__container_name":"unify-query","_value_":4182,"_timestamp_":1744672740000},{"__ext__bk_46__container_name":"unify-query","_value_":3210,"_timestamp_":1744672770000},{"__ext__bk_46__container_name":"unify-query","_value_":6383,"_timestamp_":1744672800000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744672830000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744672860000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744672890000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744672920000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744672950000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744672980000},{"__ext__bk_46__container_name":"unify-query","_value_":3534,"_timestamp_":1744673010000},{"__ext__bk_46__container_name":"unify-query","_value_":4212,"_timestamp_":1744673040000},{"__ext__bk_46__container_name":"unify-query","_value_":3380,"_timestamp_":1744673070000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744673100000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744673130000},{"__ext__bk_46__container_name":"unify-query","_value_":4120,"_timestamp_":1744673160000},{"__ext__bk_46__container_name":"unify-query","_value_":3526,"_timestamp_":1744673190000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744673220000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744673250000},{"__ext__bk_46__container_name":"unify-query","_value_":4370,"_timestamp_":1744673280000},{"__ext__bk_46__container_name":"unify-query","_value_":3462,"_timestamp_":1744673310000},{"__ext__bk_46__container_name":"unify-query","_value_":4223,"_timestamp_":1744673340000},{"__ext__bk_46__container_name":"unify-query","_value_":3564,"_timestamp_":1744673370000},{"__ext__bk_46__container_name":"unify-query","_value_":12072,"_timestamp_":1744673400000},{"__ext__bk_46__container_name":"unify-query","_value_":17986,"_timestamp_":1744673430000},{"__ext__bk_46__container_name":"unify-query","_value_":4089,"_timestamp_":1744673460000},{"__ext__bk_46__container_name":"unify-query","_value_":12000,"_timestamp_":1744673490000},{"__ext__bk_46__container_name":"unify-query","_value_":4790,"_timestamp_":1744673520000},{"__ext__bk_46__container_name":"unify-query","_value_":3637,"_timestamp_":1744673550000},{"__ext__bk_46__container_name":"unify-query","_value_":4177,"_timestamp_":1744673580000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744673610000},{"__ext__bk_46__container_name":"unify-query","_value_":4465,"_timestamp_":1744673640000},{"__ext__bk_46__container_name":"unify-query","_value_":3627,"_timestamp_":1744673670000},{"__ext__bk_46__container_name":"unify-query","_value_":4131,"_timestamp_":1744673700000},{"__ext__bk_46__container_name":"unify-query","_value_":3396,"_timestamp_":1744673730000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744673760000},{"__ext__bk_46__container_name":"unify-query","_value_":3638,"_timestamp_":1744673790000},{"__ext__bk_46__container_name":"unify-query","_value_":4093,"_timestamp_":1744673820000},{"__ext__bk_46__container_name":"unify-query","_value_":3584,"_timestamp_":1744673850000},{"__ext__bk_46__container_name":"unify-query","_value_":4082,"_timestamp_":1744673880000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744673910000},{"__ext__bk_46__container_name":"unify-query","_value_":4051,"_timestamp_":1744673940000},{"__ext__bk_46__container_name":"unify-query","_value_":3354,"_timestamp_":1744673970000},{"__ext__bk_46__container_name":"unify-query","_value_":6296,"_timestamp_":1744674000000},{"__ext__bk_46__container_name":"unify-query","_value_":3473,"_timestamp_":1744674030000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744674060000},{"__ext__bk_46__container_name":"unify-query","_value_":3793,"_timestamp_":1744674090000},{"__ext__bk_46__container_name":"unify-query","_value_":4391,"_timestamp_":1744674120000},{"__ext__bk_46__container_name":"unify-query","_value_":3836,"_timestamp_":1744674150000},{"__ext__bk_46__container_name":"unify-query","_value_":4190,"_timestamp_":1744674180000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674210000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744674240000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744674270000},{"__ext__bk_46__container_name":"unify-query","_value_":4964,"_timestamp_":1744674300000},{"__ext__bk_46__container_name":"unify-query","_value_":3455,"_timestamp_":1744674330000},{"__ext__bk_46__container_name":"unify-query","_value_":4116,"_timestamp_":1744674360000},{"__ext__bk_46__container_name":"unify-query","_value_":3250,"_timestamp_":1744674390000},{"__ext__bk_46__container_name":"unify-query","_value_":4494,"_timestamp_":1744674420000},{"__ext__bk_46__container_name":"unify-query","_value_":3326,"_timestamp_":1744674450000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744674480000},{"__ext__bk_46__container_name":"unify-query","_value_":3580,"_timestamp_":1744674510000},{"__ext__bk_46__container_name":"unify-query","_value_":4368,"_timestamp_":1744674540000},{"__ext__bk_46__container_name":"unify-query","_value_":3685,"_timestamp_":1744674570000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744674600000},{"__ext__bk_46__container_name":"unify-query","_value_":3699,"_timestamp_":1744674630000},{"__ext__bk_46__container_name":"unify-query","_value_":4513,"_timestamp_":1744674660000},{"__ext__bk_46__container_name":"unify-query","_value_":3729,"_timestamp_":1744674690000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744674720000},{"__ext__bk_46__container_name":"unify-query","_value_":3639,"_timestamp_":1744674750000},{"__ext__bk_46__container_name":"unify-query","_value_":4018,"_timestamp_":1744674780000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744674810000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744674840000},{"__ext__bk_46__container_name":"unify-query","_value_":3389,"_timestamp_":1744674870000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744674900000},{"__ext__bk_46__container_name":"unify-query","_value_":3540,"_timestamp_":1744674930000},{"__ext__bk_46__container_name":"unify-query","_value_":4106,"_timestamp_":1744674960000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674990000},{"__ext__bk_46__container_name":"unify-query","_value_":4268,"_timestamp_":1744675020000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744675050000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744675080000},{"__ext__bk_46__container_name":"unify-query","_value_":3511,"_timestamp_":1744675110000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744675140000},{"__ext__bk_46__container_name":"unify-query","_value_":3573,"_timestamp_":1744675170000},{"__ext__bk_46__container_name":"unify-query","_value_":17095,"_timestamp_":1744675200000},{"__ext__bk_46__container_name":"unify-query","_value_":14907,"_timestamp_":1744675230000},{"__ext__bk_46__container_name":"unify-query","_value_":6455,"_timestamp_":1744675260000},{"__ext__bk_46__container_name":"unify-query","_value_":9818,"_timestamp_":1744675290000},{"__ext__bk_46__container_name":"unify-query","_value_":5253,"_timestamp_":1744675320000},{"__ext__bk_46__container_name":"unify-query","_value_":3567,"_timestamp_":1744675350000},{"__ext__bk_46__container_name":"unify-query","_value_":4047,"_timestamp_":1744675380000},{"__ext__bk_46__container_name":"unify-query","_value_":3342,"_timestamp_":1744675410000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744675440000},{"__ext__bk_46__container_name":"unify-query","_value_":3394,"_timestamp_":1744675470000},{"__ext__bk_46__container_name":"unify-query","_value_":4260,"_timestamp_":1744675500000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744675530000},{"__ext__bk_46__container_name":"unify-query","_value_":4341,"_timestamp_":1744675560000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744675590000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744675620000},{"__ext__bk_46__container_name":"unify-query","_value_":3519,"_timestamp_":1744675650000},{"__ext__bk_46__container_name":"unify-query","_value_":4143,"_timestamp_":1744675680000},{"__ext__bk_46__container_name":"unify-query","_value_":3630,"_timestamp_":1744675710000},{"__ext__bk_46__container_name":"unify-query","_value_":4042,"_timestamp_":1744675740000},{"__ext__bk_46__container_name":"unify-query","_value_":3653,"_timestamp_":1744675770000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744675800000},{"__ext__bk_46__container_name":"unify-query","_value_":3688,"_timestamp_":1744675830000},{"__ext__bk_46__container_name":"unify-query","_value_":4450,"_timestamp_":1744675860000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744675890000},{"__ext__bk_46__container_name":"unify-query","_value_":4864,"_timestamp_":1744675920000},{"__ext__bk_46__container_name":"unify-query","_value_":3629,"_timestamp_":1744675950000},{"__ext__bk_46__container_name":"unify-query","_value_":4127,"_timestamp_":1744675980000},{"__ext__bk_46__container_name":"unify-query","_value_":3424,"_timestamp_":1744676010000},{"__ext__bk_46__container_name":"unify-query","_value_":4267,"_timestamp_":1744676040000},{"__ext__bk_46__container_name":"unify-query","_value_":3328,"_timestamp_":1744676070000},{"__ext__bk_46__container_name":"unify-query","_value_":5128,"_timestamp_":1744676100000},{"__ext__bk_46__container_name":"unify-query","_value_":3657,"_timestamp_":1744676130000},{"__ext__bk_46__container_name":"unify-query","_value_":4185,"_timestamp_":1744676160000},{"__ext__bk_46__container_name":"unify-query","_value_":3336,"_timestamp_":1744676190000},{"__ext__bk_46__container_name":"unify-query","_value_":4532,"_timestamp_":1744676220000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744676250000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744676280000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744676310000},{"__ext__bk_46__container_name":"unify-query","_value_":4463,"_timestamp_":1744676340000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676370000},{"__ext__bk_46__container_name":"unify-query","_value_":6064,"_timestamp_":1744676400000},{"__ext__bk_46__container_name":"unify-query","_value_":3292,"_timestamp_":1744676430000},{"__ext__bk_46__container_name":"unify-query","_value_":4858,"_timestamp_":1744676460000},{"__ext__bk_46__container_name":"unify-query","_value_":3543,"_timestamp_":1744676490000},{"__ext__bk_46__container_name":"unify-query","_value_":4620,"_timestamp_":1744676520000},{"__ext__bk_46__container_name":"unify-query","_value_":3750,"_timestamp_":1744676550000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744676580000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744676610000},{"__ext__bk_46__container_name":"unify-query","_value_":4152,"_timestamp_":1744676640000},{"__ext__bk_46__container_name":"unify-query","_value_":3550,"_timestamp_":1744676670000},{"__ext__bk_46__container_name":"unify-query","_value_":4011,"_timestamp_":1744676700000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676730000},{"__ext__bk_46__container_name":"unify-query","_value_":4050,"_timestamp_":1744676760000},{"__ext__bk_46__container_name":"unify-query","_value_":3118,"_timestamp_":1744676790000},{"__ext__bk_46__container_name":"unify-query","_value_":4628,"_timestamp_":1744676820000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744676850000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744676880000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744676910000},{"__ext__bk_46__container_name":"unify-query","_value_":4160,"_timestamp_":1744676940000},{"__ext__bk_46__container_name":"unify-query","_value_":3662,"_timestamp_":1744676970000},{"__ext__bk_46__container_name":"unify-query","_value_":11392,"_timestamp_":1744677000000},{"__ext__bk_46__container_name":"unify-query","_value_":18649,"_timestamp_":1744677030000},{"__ext__bk_46__container_name":"unify-query","_value_":7107,"_timestamp_":1744677060000},{"__ext__bk_46__container_name":"unify-query","_value_":9213,"_timestamp_":1744677090000},{"__ext__bk_46__container_name":"unify-query","_value_":4235,"_timestamp_":1744677120000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744677150000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744677180000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744677210000},{"__ext__bk_46__container_name":"unify-query","_value_":4233,"_timestamp_":1744677240000},{"__ext__bk_46__container_name":"unify-query","_value_":3440,"_timestamp_":1744677270000},{"__ext__bk_46__container_name":"unify-query","_value_":4383,"_timestamp_":1744677300000},{"__ext__bk_46__container_name":"unify-query","_value_":3507,"_timestamp_":1744677330000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744677360000},{"__ext__bk_46__container_name":"unify-query","_value_":3197,"_timestamp_":1744677390000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744677420000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744677450000},{"__ext__bk_46__container_name":"unify-query","_value_":4421,"_timestamp_":1744677480000},{"__ext__bk_46__container_name":"unify-query","_value_":2998,"_timestamp_":1744677510000},{"__ext__bk_46__container_name":"unify-query","_value_":4700,"_timestamp_":1744677540000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744677570000},{"__ext__bk_46__container_name":"unify-query","_value_":5781,"_timestamp_":1744677600000},{"__ext__bk_46__container_name":"unify-query","_value_":3734,"_timestamp_":1744677630000},{"__ext__bk_46__container_name":"unify-query","_value_":4510,"_timestamp_":1744677660000},{"__ext__bk_46__container_name":"unify-query","_value_":3752,"_timestamp_":1744677690000},{"__ext__bk_46__container_name":"unify-query","_value_":4447,"_timestamp_":1744677720000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744677750000},{"__ext__bk_46__container_name":"unify-query","_value_":4187,"_timestamp_":1744677780000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744677810000},{"__ext__bk_46__container_name":"unify-query","_value_":3900,"_timestamp_":1744677840000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744677870000},{"__ext__bk_46__container_name":"unify-query","_value_":4863,"_timestamp_":1744677900000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744677930000},{"__ext__bk_46__container_name":"unify-query","_value_":4335,"_timestamp_":1744677960000},{"__ext__bk_46__container_name":"unify-query","_value_":3533,"_timestamp_":1744677990000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744678020000},{"__ext__bk_46__container_name":"unify-query","_value_":3556,"_timestamp_":1744678050000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744678080000},{"__ext__bk_46__container_name":"unify-query","_value_":3664,"_timestamp_":1744678110000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744678140000},{"__ext__bk_46__container_name":"unify-query","_value_":3222,"_timestamp_":1744678170000},{"__ext__bk_46__container_name":"unify-query","_value_":4750,"_timestamp_":1744678200000},{"__ext__bk_46__container_name":"unify-query","_value_":3546,"_timestamp_":1744678230000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744678260000},{"__ext__bk_46__container_name":"unify-query","_value_":3702,"_timestamp_":1744678290000},{"__ext__bk_46__container_name":"unify-query","_value_":4564,"_timestamp_":1744678320000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744678350000},{"__ext__bk_46__container_name":"unify-query","_value_":4130,"_timestamp_":1744678380000},{"__ext__bk_46__container_name":"unify-query","_value_":3412,"_timestamp_":1744678410000},{"__ext__bk_46__container_name":"unify-query","_value_":4614,"_timestamp_":1744678440000},{"__ext__bk_46__container_name":"unify-query","_value_":3522,"_timestamp_":1744678470000},{"__ext__bk_46__container_name":"unify-query","_value_":4148,"_timestamp_":1744678500000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744678530000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744678560000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744678590000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744678620000},{"__ext__bk_46__container_name":"unify-query","_value_":3529,"_timestamp_":1744678650000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744678680000},{"__ext__bk_46__container_name":"unify-query","_value_":3487,"_timestamp_":1744678710000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744678740000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744678770000},{"__ext__bk_46__container_name":"unify-query","_value_":7230,"_timestamp_":1744678800000},{"__ext__bk_46__container_name":"unify-query","_value_":3818,"_timestamp_":1744678830000},{"__ext__bk_46__container_name":"unify-query","_value_":11924,"_timestamp_":1744678860000},{"__ext__bk_46__container_name":"unify-query","_value_":27269,"_timestamp_":1744678890000},{"__ext__bk_46__container_name":"unify-query","_value_":5073,"_timestamp_":1744678920000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744678950000},{"__ext__bk_46__container_name":"unify-query","_value_":4474,"_timestamp_":1744678980000},{"__ext__bk_46__container_name":"unify-query","_value_":3536,"_timestamp_":1744679010000},{"__ext__bk_46__container_name":"unify-query","_value_":4525,"_timestamp_":1744679040000},{"__ext__bk_46__container_name":"unify-query","_value_":3503,"_timestamp_":1744679070000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744679100000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744679130000},{"__ext__bk_46__container_name":"unify-query","_value_":4259,"_timestamp_":1744679160000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744679190000},{"__ext__bk_46__container_name":"unify-query","_value_":4218,"_timestamp_":1744679220000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744679250000},{"__ext__bk_46__container_name":"unify-query","_value_":4417,"_timestamp_":1744679280000},{"__ext__bk_46__container_name":"unify-query","_value_":3730,"_timestamp_":1744679310000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744679340000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744679370000},{"__ext__bk_46__container_name":"unify-query","_value_":4849,"_timestamp_":1744679400000},{"__ext__bk_46__container_name":"unify-query","_value_":3803,"_timestamp_":1744679430000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744679460000},{"__ext__bk_46__container_name":"unify-query","_value_":3674,"_timestamp_":1744679490000},{"__ext__bk_46__container_name":"unify-query","_value_":4727,"_timestamp_":1744679520000},{"__ext__bk_46__container_name":"unify-query","_value_":3926,"_timestamp_":1744679550000},{"__ext__bk_46__container_name":"unify-query","_value_":4173,"_timestamp_":1744679580000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744679610000},{"__ext__bk_46__container_name":"unify-query","_value_":4968,"_timestamp_":1744679640000},{"__ext__bk_46__container_name":"unify-query","_value_":3432,"_timestamp_":1744679670000},{"__ext__bk_46__container_name":"unify-query","_value_":5059,"_timestamp_":1744679700000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744679730000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744679760000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744679790000},{"__ext__bk_46__container_name":"unify-query","_value_":4436,"_timestamp_":1744679820000},{"__ext__bk_46__container_name":"unify-query","_value_":5299,"_timestamp_":1744679850000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744679880000},{"__ext__bk_46__container_name":"unify-query","_value_":3861,"_timestamp_":1744679910000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744679940000},{"__ext__bk_46__container_name":"unify-query","_value_":3711,"_timestamp_":1744679970000},{"__ext__bk_46__container_name":"unify-query","_value_":6021,"_timestamp_":1744680000000},{"__ext__bk_46__container_name":"unify-query","_value_":3942,"_timestamp_":1744680030000},{"__ext__bk_46__container_name":"unify-query","_value_":4800,"_timestamp_":1744680060000},{"__ext__bk_46__container_name":"unify-query","_value_":3681,"_timestamp_":1744680090000},{"__ext__bk_46__container_name":"unify-query","_value_":4592,"_timestamp_":1744680120000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744680150000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744680180000},{"__ext__bk_46__container_name":"unify-query","_value_":3490,"_timestamp_":1744680210000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744680240000},{"__ext__bk_46__container_name":"unify-query","_value_":4009,"_timestamp_":1744680270000},{"__ext__bk_46__container_name":"unify-query","_value_":4837,"_timestamp_":1744680300000},{"__ext__bk_46__container_name":"unify-query","_value_":3227,"_timestamp_":1744680330000},{"__ext__bk_46__container_name":"unify-query","_value_":4531,"_timestamp_":1744680360000},{"__ext__bk_46__container_name":"unify-query","_value_":2888,"_timestamp_":1744680390000},{"__ext__bk_46__container_name":"unify-query","_value_":5083,"_timestamp_":1744680420000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744680450000},{"__ext__bk_46__container_name":"unify-query","_value_":4207,"_timestamp_":1744680480000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744680510000},{"__ext__bk_46__container_name":"unify-query","_value_":4482,"_timestamp_":1744680540000},{"__ext__bk_46__container_name":"unify-query","_value_":3110,"_timestamp_":1744680570000},{"__ext__bk_46__container_name":"unify-query","_value_":13551,"_timestamp_":1744680600000},{"__ext__bk_46__container_name":"unify-query","_value_":17159,"_timestamp_":1744680630000},{"__ext__bk_46__container_name":"unify-query","_value_":6284,"_timestamp_":1744680660000},{"__ext__bk_46__container_name":"unify-query","_value_":9924,"_timestamp_":1744680690000},{"__ext__bk_46__container_name":"unify-query","_value_":4547,"_timestamp_":1744680720000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744680750000},{"__ext__bk_46__container_name":"unify-query","_value_":4312,"_timestamp_":1744680780000},{"__ext__bk_46__container_name":"unify-query","_value_":3689,"_timestamp_":1744680810000},{"__ext__bk_46__container_name":"unify-query","_value_":4680,"_timestamp_":1744680840000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744680870000},{"__ext__bk_46__container_name":"unify-query","_value_":4886,"_timestamp_":1744680900000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744680930000},{"__ext__bk_46__container_name":"unify-query","_value_":4810,"_timestamp_":1744680960000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744680990000},{"__ext__bk_46__container_name":"unify-query","_value_":4594,"_timestamp_":1744681020000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744681050000},{"__ext__bk_46__container_name":"unify-query","_value_":4562,"_timestamp_":1744681080000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744681110000},{"__ext__bk_46__container_name":"unify-query","_value_":5243,"_timestamp_":1744681140000},{"__ext__bk_46__container_name":"unify-query","_value_":5135,"_timestamp_":1744681170000},{"__ext__bk_46__container_name":"unify-query","_value_":6671,"_timestamp_":1744681200000},{"__ext__bk_46__container_name":"unify-query","_value_":3806,"_timestamp_":1744681230000},{"__ext__bk_46__container_name":"unify-query","_value_":4535,"_timestamp_":1744681260000},{"__ext__bk_46__container_name":"unify-query","_value_":3721,"_timestamp_":1744681290000},{"__ext__bk_46__container_name":"unify-query","_value_":4799,"_timestamp_":1744681320000},{"__ext__bk_46__container_name":"unify-query","_value_":3909,"_timestamp_":1744681350000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744681380000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744681410000},{"__ext__bk_46__container_name":"unify-query","_value_":4359,"_timestamp_":1744681440000},{"__ext__bk_46__container_name":"unify-query","_value_":4063,"_timestamp_":1744681470000},{"__ext__bk_46__container_name":"unify-query","_value_":5231,"_timestamp_":1744681500000},{"__ext__bk_46__container_name":"unify-query","_value_":3778,"_timestamp_":1744681530000},{"__ext__bk_46__container_name":"unify-query","_value_":4684,"_timestamp_":1744681560000},{"__ext__bk_46__container_name":"unify-query","_value_":4072,"_timestamp_":1744681590000},{"__ext__bk_46__container_name":"unify-query","_value_":5029,"_timestamp_":1744681620000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744681650000},{"__ext__bk_46__container_name":"unify-query","_value_":4670,"_timestamp_":1744681680000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744681710000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744681740000},{"__ext__bk_46__container_name":"unify-query","_value_":3041,"_timestamp_":1744681770000},{"__ext__bk_46__container_name":"unify-query","_value_":5043,"_timestamp_":1744681800000},{"__ext__bk_46__container_name":"unify-query","_value_":3530,"_timestamp_":1744681830000},{"__ext__bk_46__container_name":"unify-query","_value_":6807,"_timestamp_":1744681860000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744681890000},{"__ext__bk_46__container_name":"unify-query","_value_":6841,"_timestamp_":1744681920000},{"__ext__bk_46__container_name":"unify-query","_value_":4519,"_timestamp_":1744681950000},{"__ext__bk_46__container_name":"unify-query","_value_":6617,"_timestamp_":1744681980000},{"__ext__bk_46__container_name":"unify-query","_value_":4633,"_timestamp_":1744682010000},{"__ext__bk_46__container_name":"unify-query","_value_":5997,"_timestamp_":1744682040000},{"__ext__bk_46__container_name":"unify-query","_value_":4446,"_timestamp_":1744682070000},{"__ext__bk_46__container_name":"unify-query","_value_":5569,"_timestamp_":1744682100000},{"__ext__bk_46__container_name":"unify-query","_value_":4324,"_timestamp_":1744682130000},{"__ext__bk_46__container_name":"unify-query","_value_":5354,"_timestamp_":1744682160000},{"__ext__bk_46__container_name":"unify-query","_value_":7245,"_timestamp_":1744682190000},{"__ext__bk_46__container_name":"unify-query","_value_":5258,"_timestamp_":1744682220000},{"__ext__bk_46__container_name":"unify-query","_value_":4296,"_timestamp_":1744682250000},{"__ext__bk_46__container_name":"unify-query","_value_":5349,"_timestamp_":1744682280000},{"__ext__bk_46__container_name":"unify-query","_value_":4479,"_timestamp_":1744682310000},{"__ext__bk_46__container_name":"unify-query","_value_":5127,"_timestamp_":1744682340000},{"__ext__bk_46__container_name":"unify-query","_value_":4006,"_timestamp_":1744682370000},{"__ext__bk_46__container_name":"unify-query","_value_":19058,"_timestamp_":1744682400000},{"__ext__bk_46__container_name":"unify-query","_value_":14501,"_timestamp_":1744682430000},{"__ext__bk_46__container_name":"unify-query","_value_":3810,"_timestamp_":1744682460000},{"__ext__bk_46__container_name":"unify-query","_value_":12368,"_timestamp_":1744682490000},{"__ext__bk_46__container_name":"unify-query","_value_":6976,"_timestamp_":1744682520000},{"__ext__bk_46__container_name":"unify-query","_value_":4399,"_timestamp_":1744682550000},{"__ext__bk_46__container_name":"unify-query","_value_":5482,"_timestamp_":1744682580000},{"__ext__bk_46__container_name":"unify-query","_value_":4524,"_timestamp_":1744682610000},{"__ext__bk_46__container_name":"unify-query","_value_":5478,"_timestamp_":1744682640000},{"__ext__bk_46__container_name":"unify-query","_value_":4920,"_timestamp_":1744682670000},{"__ext__bk_46__container_name":"unify-query","_value_":5347,"_timestamp_":1744682700000},{"__ext__bk_46__container_name":"unify-query","_value_":4427,"_timestamp_":1744682730000},{"__ext__bk_46__container_name":"unify-query","_value_":5102,"_timestamp_":1744682760000},{"__ext__bk_46__container_name":"unify-query","_value_":4441,"_timestamp_":1744682790000},{"__ext__bk_46__container_name":"unify-query","_value_":5596,"_timestamp_":1744682820000},{"__ext__bk_46__container_name":"unify-query","_value_":4888,"_timestamp_":1744682850000},{"__ext__bk_46__container_name":"unify-query","_value_":5306,"_timestamp_":1744682880000},{"__ext__bk_46__container_name":"unify-query","_value_":4825,"_timestamp_":1744682910000},{"__ext__bk_46__container_name":"unify-query","_value_":5897,"_timestamp_":1744682940000},{"__ext__bk_46__container_name":"unify-query","_value_":4481,"_timestamp_":1744682970000},{"__ext__bk_46__container_name":"unify-query","_value_":6086,"_timestamp_":1744683000000},{"__ext__bk_46__container_name":"unify-query","_value_":4910,"_timestamp_":1744683030000},{"__ext__bk_46__container_name":"unify-query","_value_":5676,"_timestamp_":1744683060000},{"__ext__bk_46__container_name":"unify-query","_value_":3626,"_timestamp_":1744683090000},{"__ext__bk_46__container_name":"unify-query","_value_":6929,"_timestamp_":1744683120000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744683150000},{"__ext__bk_46__container_name":"unify-query","_value_":5525,"_timestamp_":1744683180000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744683210000},{"__ext__bk_46__container_name":"unify-query","_value_":5617,"_timestamp_":1744683240000},{"__ext__bk_46__container_name":"unify-query","_value_":4503,"_timestamp_":1744683270000},{"__ext__bk_46__container_name":"unify-query","_value_":6328,"_timestamp_":1744683300000},{"__ext__bk_46__container_name":"unify-query","_value_":4557,"_timestamp_":1744683330000},{"__ext__bk_46__container_name":"unify-query","_value_":5356,"_timestamp_":1744683360000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744683390000},{"__ext__bk_46__container_name":"unify-query","_value_":5335,"_timestamp_":1744683420000},{"__ext__bk_46__container_name":"unify-query","_value_":4640,"_timestamp_":1744683450000},{"__ext__bk_46__container_name":"unify-query","_value_":5399,"_timestamp_":1744683480000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744683510000},{"__ext__bk_46__container_name":"unify-query","_value_":5415,"_timestamp_":1744683540000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744683570000},{"__ext__bk_46__container_name":"unify-query","_value_":6949,"_timestamp_":1744683600000},{"__ext__bk_46__container_name":"unify-query","_value_":4574,"_timestamp_":1744683630000},{"__ext__bk_46__container_name":"unify-query","_value_":5757,"_timestamp_":1744683660000},{"__ext__bk_46__container_name":"unify-query","_value_":4669,"_timestamp_":1744683690000},{"__ext__bk_46__container_name":"unify-query","_value_":5706,"_timestamp_":1744683720000},{"__ext__bk_46__container_name":"unify-query","_value_":4472,"_timestamp_":1744683750000},{"__ext__bk_46__container_name":"unify-query","_value_":5386,"_timestamp_":1744683780000},{"__ext__bk_46__container_name":"unify-query","_value_":4490,"_timestamp_":1744683810000},{"__ext__bk_46__container_name":"unify-query","_value_":5104,"_timestamp_":1744683840000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744683870000},{"__ext__bk_46__container_name":"unify-query","_value_":5979,"_timestamp_":1744683900000},{"__ext__bk_46__container_name":"unify-query","_value_":4853,"_timestamp_":1744683930000},{"__ext__bk_46__container_name":"unify-query","_value_":6691,"_timestamp_":1744683960000},{"__ext__bk_46__container_name":"unify-query","_value_":4572,"_timestamp_":1744683990000},{"__ext__bk_46__container_name":"unify-query","_value_":5554,"_timestamp_":1744684020000},{"__ext__bk_46__container_name":"unify-query","_value_":5244,"_timestamp_":1744684050000},{"__ext__bk_46__container_name":"unify-query","_value_":5392,"_timestamp_":1744684080000},{"__ext__bk_46__container_name":"unify-query","_value_":4550,"_timestamp_":1744684110000},{"__ext__bk_46__container_name":"unify-query","_value_":520,"_timestamp_":1744684140000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":52,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":8,"connect_db":55,"match_query_routing_rule":0,"check_permission":73,"check_query_semantic":0,"pick_valid_storage":1},"total_record_size":269248,"timetaken":0.191,"result_schema":[{"field_type":"string","field_name":"__c0","field_alias":"__ext__bk_46__container_name","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1},{"field_type":"long","field_name":"__c2","field_alias":"_timestamp_","field_index":2}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + }) + + for i, c := range map[string]struct { + queryTs *structured.QueryTs + result string + }{ + "查询 1 条原始数据，按照字段正向排序": { + queryTs: &structured.QueryTs{ + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(tableID), + FieldName: "gseIndex", + Limit: 1, + From: 0, + ReferenceName: "a", + }, + }, + OrderBy: structured.OrderBy{ + "_value", + }, + MetricMerge: "a", + Start: strconv.FormatInt(defaultStart.Unix(), 10), + End: strconv.FormatInt(defaultEnd.Unix(), 10), + Instant: false, + SpaceUid: spaceUid, + }, + result: `{ + "series" : [ { + "name" : "_result0", + "metric_name" : "", + "columns" : [ "_time", "_value" ], + "types" : [ "float", "float" ], + "group_keys" : [ "__ext.container_id", "__ext.container_image", "__ext.container_name", "__ext.io_kubernetes_pod", "__ext.io_kubernetes_pod_ip", "__ext.io_kubernetes_pod_namespace", "__ext.io_kubernetes_pod_uid", "__ext.io_kubernetes_workload_name", "__ext.io_kubernetes_workload_type", "__name__", "__shard_key__", "cloudid", "dteventtime", "dteventtimestamp", "file", "gseindex", "iterationindex", "level", "localtime", "log", "message", "path", "report_time", "serverip", "thedate", "time", "trace_id" ], + "group_values" : [ "375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2", "sha256:3a0506f06f1467e93c3a582203aac1a7501e77091572ec9612ddeee4a4dbbdb8", "unify-query", "bk-datalink-unify-query-6df8bcc4c9-rk4sc", "127.0.0.1", "blueking", "558c5b17-b221-47e1-aa66-036cc9b43e2a", "bk-datalink-unify-query-6df8bcc4c9", "ReplicaSet", "bklog:result_table:doris:gseIndex", "29077703953", "0", "", "1744662180000", "http/handler.go:320", "2450131", "19", "info", "", "2025-04-14T20:22:59.982Z\tinfo\thttp/handler.go:320\t[5108397435e997364f8dc1251533e65e] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}", " header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}", "/var/host/data/bcs/lib/docker/containers/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2-json.log", "2025-04-14T20:22:59.982Z", "30.189.37.111", "20250415", "1744662180000", "5108397435e997364f8dc1251533e65e" ], + "values" : [ [ 1744662480000, 2450131 ] ] + } ] +}`, + }, + "根据维度 __ext.container_name 进行 count 聚合，同时用值正向排序": { + queryTs: &structured.QueryTs{ + QueryList: []*structured.Query{ + { + DataSource: structured.BkLog, + TableID: structured.TableID(tableID), + FieldName: "gseIndex", + ReferenceName: "a", + TimeAggregation: structured.TimeAggregation{ + Function: "count_over_time", + Window: "30s", + }, + AggregateMethodList: structured.AggregateMethodList{ + { + Method: "sum", + Dimensions: []string{"__ext.container_name"}, + }, + { + Method: "topk", + VArgsList: []interface{}{ + 5, + }, + }, + }, + }, + }, + OrderBy: structured.OrderBy{ + "_value", + }, + MetricMerge: "a", + Start: strconv.FormatInt(defaultStart.Unix(), 10), + End: strconv.FormatInt(defaultEnd.Unix(), 10), + Instant: false, + SpaceUid: spaceUid, + Step: "30s", + }, + result: `{ + "series" : [ { + "name" : "_result0", + "metric_name" : "", + "columns" : [ "_time", "_value" ], + "types" : [ "float", "float" ], + "group_keys" : [ "__ext.container_name" ], + "group_values" : [ "unify-query" ], + "values" : [ [ 1744662510000, 3684 ], [ 1744662540000, 4012 ], [ 1744662570000, 3671 ], [ 1744662600000, 17092 ], [ 1744662630000, 12881 ], [ 1744662660000, 5902 ], [ 1744662690000, 10443 ], [ 1744662720000, 4388 ], [ 1744662750000, 3357 ], [ 1744662780000, 4381 ], [ 1744662810000, 3683 ], [ 1744662840000, 4353 ], [ 1744662870000, 3441 ], [ 1744662900000, 4251 ], [ 1744662930000, 3476 ], [ 1744662960000, 4036 ], [ 1744662990000, 3549 ], [ 1744663020000, 4351 ], [ 1744663050000, 3651 ], [ 1744663080000, 4096 ], [ 1744663110000, 3618 ], [ 1744663140000, 4100 ], [ 1744663170000, 3622 ], [ 1744663200000, 6044 ], [ 1744663230000, 3766 ], [ 1744663260000, 4461 ], [ 1744663290000, 3783 ], [ 1744663320000, 4559 ], [ 1744663350000, 3634 ], [ 1744663380000, 3869 ], [ 1744663410000, 3249 ], [ 1744663440000, 4473 ], [ 1744663470000, 3514 ], [ 1744663500000, 4923 ], [ 1744663530000, 3379 ], [ 1744663560000, 4489 ], [ 1744663590000, 3411 ], [ 1744663620000, 4374 ], [ 1744663650000, 3370 ], [ 1744663680000, 4310 ], [ 1744663710000, 3609 ], [ 1744663740000, 4318 ], [ 1744663770000, 3570 ], [ 1744663800000, 4334 ], [ 1744663830000, 3767 ], [ 1744663860000, 4455 ], [ 1744663890000, 3703 ], [ 1744663920000, 4511 ], [ 1744663950000, 3667 ], [ 1744663980000, 3998 ], [ 1744664010000, 3579 ], [ 1744664040000, 4156 ], [ 1744664070000, 3340 ], [ 1744664100000, 4344 ], [ 1744664130000, 3590 ], [ 1744664160000, 4161 ], [ 1744664190000, 3484 ], [ 1744664220000, 4273 ], [ 1744664250000, 3494 ], [ 1744664280000, 4230 ], [ 1744664310000, 3619 ], [ 1744664340000, 4013 ], [ 1744664370000, 3565 ], [ 1744664400000, 18144 ], [ 1744664430000, 13615 ], [ 1744664460000, 3178 ], [ 1744664490000, 13044 ], [ 1744664520000, 4767 ], [ 1744664550000, 3528 ], [ 1744664580000, 4316 ], [ 1744664610000, 3317 ], [ 1744664640000, 4395 ], [ 1744664670000, 3599 ], [ 1744664700000, 4149 ], [ 1744664730000, 3474 ], [ 1744664760000, 4201 ], [ 1744664790000, 3384 ], [ 1744664820000, 4442 ], [ 1744664850000, 3559 ], [ 1744664880000, 4166 ], [ 1744664910000, 3438 ], [ 1744664940000, 4244 ], [ 1744664970000, 3640 ], [ 1744665000000, 4305 ], [ 1744665030000, 3771 ], [ 1744665060000, 4485 ], [ 1744665090000, 3842 ], [ 1744665120000, 4423 ], [ 1744665150000, 3610 ], [ 1744665180000, 4125 ], [ 1744665210000, 3500 ], [ 1744665240000, 4252 ], [ 1744665270000, 3427 ], [ 1744665300000, 5089 ], [ 1744665330000, 3450 ], [ 1744665360000, 4349 ], [ 1744665390000, 3188 ], [ 1744665420000, 4556 ], [ 1744665450000, 3372 ], [ 1744665480000, 4408 ], [ 1744665510000, 3445 ], [ 1744665540000, 4213 ], [ 1744665570000, 3408 ], [ 1744665600000, 6235 ], [ 1744665630000, 3641 ], [ 1744665660000, 4577 ], [ 1744665690000, 3719 ], [ 1744665720000, 4548 ], [ 1744665750000, 3420 ], [ 1744665780000, 4246 ], [ 1744665810000, 3359 ], [ 1744665840000, 4332 ], [ 1744665870000, 3422 ], [ 1744665900000, 4229 ], [ 1744665930000, 3610 ], [ 1744665960000, 4119 ], [ 1744665990000, 3570 ], [ 1744666020000, 4144 ], [ 1744666050000, 3302 ], [ 1744666080000, 4398 ], [ 1744666110000, 3559 ], [ 1744666140000, 4097 ], [ 1744666170000, 3315 ], [ 1744666200000, 16721 ], [ 1744666230000, 13631 ], [ 1744666260000, 2982 ], [ 1744666290000, 11858 ], [ 1744666320000, 5515 ], [ 1744666350000, 2869 ], [ 1744666380000, 4795 ], [ 1744666410000, 3603 ], [ 1744666440000, 4204 ], [ 1744666470000, 3264 ], [ 1744666500000, 4377 ], [ 1744666530000, 3443 ], [ 1744666560000, 4307 ], [ 1744666590000, 3459 ], [ 1744666620000, 4342 ], [ 1744666650000, 3598 ], [ 1744666680000, 4052 ], [ 1744666710000, 3577 ], [ 1744666740000, 4128 ], [ 1744666770000, 3499 ], [ 1744666800000, 6209 ], [ 1744666830000, 3575 ], [ 1744666860000, 4543 ], [ 1744666890000, 3604 ], [ 1744666920000, 4579 ], [ 1744666950000, 3531 ], [ 1744666980000, 4314 ], [ 1744667010000, 3416 ], [ 1744667040000, 4320 ], [ 1744667070000, 3488 ], [ 1744667100000, 5054 ], [ 1744667130000, 3525 ], [ 1744667160000, 4313 ], [ 1744667190000, 3607 ], [ 1744667220000, 4118 ], [ 1744667250000, 3350 ], [ 1744667280000, 4280 ], [ 1744667310000, 3634 ], [ 1744667340000, 4174 ], [ 1744667370000, 3807 ], [ 1744667400000, 4358 ], [ 1744667430000, 3595 ], [ 1744667460000, 4630 ], [ 1744667490000, 3845 ], [ 1744667520000, 4361 ], [ 1744667550000, 3572 ], [ 1744667580000, 4095 ], [ 1744667610000, 3535 ], [ 1744667640000, 4200 ], [ 1744667670000, 3390 ], [ 1744667700000, 4262 ], [ 1744667730000, 3398 ], [ 1744667760000, 4320 ], [ 1744667790000, 3429 ], [ 1744667820000, 4288 ], [ 1744667850000, 3482 ], [ 1744667880000, 4166 ], [ 1744667910000, 3612 ], [ 1744667940000, 4194 ], [ 1744667970000, 3423 ], [ 1744668000000, 18203 ], [ 1744668030000, 13685 ], [ 1744668060000, 3281 ], [ 1744668090000, 12556 ], [ 1744668120000, 4893 ], [ 1744668150000, 3607 ], [ 1744668180000, 4336 ], [ 1744668210000, 3609 ], [ 1744668240000, 4097 ], [ 1744668270000, 3669 ], [ 1744668300000, 3997 ], [ 1744668330000, 3494 ], [ 1744668360000, 4172 ], [ 1744668390000, 3523 ], [ 1744668420000, 3877 ], [ 1744668450000, 3565 ], [ 1744668480000, 4230 ], [ 1744668510000, 3469 ], [ 1744668540000, 4243 ], [ 1744668570000, 3304 ], [ 1744668600000, 4690 ], [ 1744668630000, 3717 ], [ 1744668660000, 4618 ], [ 1744668690000, 3732 ], [ 1744668720000, 4477 ], [ 1744668750000, 3615 ], [ 1744668780000, 4154 ], [ 1744668810000, 3367 ], [ 1744668840000, 4193 ], [ 1744668870000, 3592 ], [ 1744668900000, 4971 ], [ 1744668930000, 3359 ], [ 1744668960000, 4540 ], [ 1744668990000, 3406 ], [ 1744669020000, 4375 ], [ 1744669050000, 3386 ], [ 1744669080000, 4281 ], [ 1744669110000, 3410 ], [ 1744669140000, 4545 ], [ 1744669170000, 3724 ], [ 1744669200000, 5903 ], [ 1744669230000, 3672 ], [ 1744669260000, 4413 ], [ 1744669290000, 3792 ], [ 1744669320000, 4422 ], [ 1744669350000, 3718 ], [ 1744669380000, 4213 ], [ 1744669410000, 3622 ], [ 1744669440000, 4043 ], [ 1744669470000, 3542 ], [ 1744669500000, 4179 ], [ 1744669530000, 3368 ], [ 1744669560000, 4354 ], [ 1744669590000, 3368 ], [ 1744669620000, 4229 ], [ 1744669650000, 3458 ], [ 1744669680000, 4310 ], [ 1744669710000, 3512 ], [ 1744669740000, 4188 ], [ 1744669770000, 3436 ], [ 1744669800000, 12171 ], [ 1744669830000, 18129 ], [ 1744669860000, 7142 ], [ 1744669890000, 9153 ], [ 1744669920000, 4566 ], [ 1744669950000, 3225 ], [ 1744669980000, 4378 ], [ 1744670010000, 3623 ], [ 1744670040000, 4266 ], [ 1744670070000, 3645 ], [ 1744670100000, 4043 ], [ 1744670130000, 3350 ], [ 1744670160000, 4333 ], [ 1744670190000, 3489 ], [ 1744670220000, 4303 ], [ 1744670250000, 3560 ], [ 1744670280000, 4121 ], [ 1744670310000, 3374 ], [ 1744670340000, 4362 ], [ 1744670370000, 3242 ], [ 1744670400000, 6416 ], [ 1744670430000, 3697 ], [ 1744670460000, 4506 ], [ 1744670490000, 3749 ], [ 1744670520000, 4587 ], [ 1744670550000, 3538 ], [ 1744670580000, 4221 ], [ 1744670610000, 3476 ], [ 1744670640000, 4227 ], [ 1744670670000, 3587 ], [ 1744670700000, 4848 ], [ 1744670730000, 3551 ], [ 1744670760000, 4068 ], [ 1744670790000, 3387 ], [ 1744670820000, 4366 ], [ 1744670850000, 3635 ], [ 1744670880000, 4256 ], [ 1744670910000, 3690 ], [ 1744670940000, 4155 ], [ 1744670970000, 3318 ], [ 1744671000000, 4661 ], [ 1744671030000, 3494 ], [ 1744671060000, 4442 ], [ 1744671090000, 3643 ], [ 1744671120000, 4755 ], [ 1744671150000, 3607 ], [ 1744671180000, 4284 ], [ 1744671210000, 3258 ], [ 1744671240000, 4453 ], [ 1744671270000, 3431 ], [ 1744671300000, 4231 ], [ 1744671330000, 3623 ], [ 1744671360000, 3907 ], [ 1744671390000, 3524 ], [ 1744671420000, 4438 ], [ 1744671450000, 3547 ], [ 1744671480000, 4033 ], [ 1744671510000, 3632 ], [ 1744671540000, 4162 ], [ 1744671570000, 3588 ], [ 1744671600000, 16444 ], [ 1744671630000, 15396 ], [ 1744671660000, 3024 ], [ 1744671690000, 12656 ], [ 1744671720000, 4733 ], [ 1744671750000, 3766 ], [ 1744671780000, 4388 ], [ 1744671810000, 3340 ], [ 1744671840000, 4487 ], [ 1744671870000, 3549 ], [ 1744671900000, 4154 ], [ 1744671930000, 3406 ], [ 1744671960000, 4314 ], [ 1744671990000, 3472 ], [ 1744672020000, 4309 ], [ 1744672050000, 3458 ], [ 1744672080000, 4191 ], [ 1744672110000, 3475 ], [ 1744672140000, 4194 ], [ 1744672170000, 3525 ], [ 1744672200000, 4445 ], [ 1744672230000, 3822 ], [ 1744672260000, 4346 ], [ 1744672290000, 3700 ], [ 1744672320000, 4615 ], [ 1744672350000, 3591 ], [ 1744672380000, 4056 ], [ 1744672410000, 3544 ], [ 1744672440000, 4188 ], [ 1744672470000, 3647 ], [ 1744672500000, 4887 ], [ 1744672530000, 3450 ], [ 1744672560000, 4302 ], [ 1744672590000, 3425 ], [ 1744672620000, 4320 ], [ 1744672650000, 3532 ], [ 1744672680000, 4282 ], [ 1744672710000, 3571 ], [ 1744672740000, 4182 ], [ 1744672770000, 3210 ], [ 1744672800000, 6383 ], [ 1744672830000, 3622 ], [ 1744672860000, 4408 ], [ 1744672890000, 3611 ], [ 1744672920000, 4795 ], [ 1744672950000, 3632 ], [ 1744672980000, 4102 ], [ 1744673010000, 3534 ], [ 1744673040000, 4212 ], [ 1744673070000, 3380 ], [ 1744673100000, 4289 ], [ 1744673130000, 3565 ], [ 1744673160000, 4120 ], [ 1744673190000, 3526 ], [ 1744673220000, 4200 ], [ 1744673250000, 3302 ], [ 1744673280000, 4370 ], [ 1744673310000, 3462 ], [ 1744673340000, 4223 ], [ 1744673370000, 3564 ], [ 1744673400000, 12072 ], [ 1744673430000, 17986 ], [ 1744673460000, 4089 ], [ 1744673490000, 12000 ], [ 1744673520000, 4790 ], [ 1744673550000, 3637 ], [ 1744673580000, 4177 ], [ 1744673610000, 3438 ], [ 1744673640000, 4465 ], [ 1744673670000, 3627 ], [ 1744673700000, 4131 ], [ 1744673730000, 3396 ], [ 1744673760000, 4395 ], [ 1744673790000, 3638 ], [ 1744673820000, 4093 ], [ 1744673850000, 3584 ], [ 1744673880000, 4082 ], [ 1744673910000, 3475 ], [ 1744673940000, 4051 ], [ 1744673970000, 3354 ], [ 1744674000000, 6296 ], [ 1744674030000, 3473 ], [ 1744674060000, 4412 ], [ 1744674090000, 3793 ], [ 1744674120000, 4391 ], [ 1744674150000, 3836 ], [ 1744674180000, 4190 ], [ 1744674210000, 3478 ], [ 1744674240000, 4230 ], [ 1744674270000, 3488 ], [ 1744674300000, 4964 ], [ 1744674330000, 3455 ], [ 1744674360000, 4116 ], [ 1744674390000, 3250 ], [ 1744674420000, 4494 ], [ 1744674450000, 3326 ], [ 1744674480000, 4590 ], [ 1744674510000, 3580 ], [ 1744674540000, 4368 ], [ 1744674570000, 3685 ], [ 1744674600000, 4381 ], [ 1744674630000, 3699 ], [ 1744674660000, 4513 ], [ 1744674690000, 3729 ], [ 1744674720000, 4500 ], [ 1744674750000, 3639 ], [ 1744674780000, 4018 ], [ 1744674810000, 3587 ], [ 1744674840000, 4168 ], [ 1744674870000, 3389 ], [ 1744674900000, 4289 ], [ 1744674930000, 3540 ], [ 1744674960000, 4106 ], [ 1744674990000, 3478 ], [ 1744675020000, 4268 ], [ 1744675050000, 3577 ], [ 1744675080000, 4087 ], [ 1744675110000, 3511 ], [ 1744675140000, 4174 ], [ 1744675170000, 3573 ], [ 1744675200000, 17095 ], [ 1744675230000, 14907 ], [ 1744675260000, 6455 ], [ 1744675290000, 9818 ], [ 1744675320000, 5253 ], [ 1744675350000, 3567 ], [ 1744675380000, 4047 ], [ 1744675410000, 3342 ], [ 1744675440000, 4605 ], [ 1744675470000, 3394 ], [ 1744675500000, 4260 ], [ 1744675530000, 3373 ], [ 1744675560000, 4341 ], [ 1744675590000, 3559 ], [ 1744675620000, 4188 ], [ 1744675650000, 3519 ], [ 1744675680000, 4143 ], [ 1744675710000, 3630 ], [ 1744675740000, 4042 ], [ 1744675770000, 3653 ], [ 1744675800000, 4358 ], [ 1744675830000, 3688 ], [ 1744675860000, 4450 ], [ 1744675890000, 3387 ], [ 1744675920000, 4864 ], [ 1744675950000, 3629 ], [ 1744675980000, 4127 ], [ 1744676010000, 3424 ], [ 1744676040000, 4267 ], [ 1744676070000, 3328 ], [ 1744676100000, 5128 ], [ 1744676130000, 3657 ], [ 1744676160000, 4185 ], [ 1744676190000, 3336 ], [ 1744676220000, 4532 ], [ 1744676250000, 3700 ], [ 1744676280000, 4174 ], [ 1744676310000, 3318 ], [ 1744676340000, 4463 ], [ 1744676370000, 3502 ], [ 1744676400000, 6064 ], [ 1744676430000, 3292 ], [ 1744676460000, 4858 ], [ 1744676490000, 3543 ], [ 1744676520000, 4620 ], [ 1744676550000, 3750 ], [ 1744676580000, 4043 ], [ 1744676610000, 3595 ], [ 1744676640000, 4152 ], [ 1744676670000, 3550 ], [ 1744676700000, 4011 ], [ 1744676730000, 3502 ], [ 1744676760000, 4050 ], [ 1744676790000, 3118 ], [ 1744676820000, 4628 ], [ 1744676850000, 3441 ], [ 1744676880000, 4366 ], [ 1744676910000, 3500 ], [ 1744676940000, 4160 ], [ 1744676970000, 3662 ], [ 1744677000000, 11392 ], [ 1744677030000, 18649 ], [ 1744677060000, 7107 ], [ 1744677090000, 9213 ], [ 1744677120000, 4235 ], [ 1744677150000, 3623 ], [ 1744677180000, 4412 ], [ 1744677210000, 3436 ], [ 1744677240000, 4233 ], [ 1744677270000, 3440 ], [ 1744677300000, 4383 ], [ 1744677330000, 3507 ], [ 1744677360000, 4288 ], [ 1744677390000, 3197 ], [ 1744677420000, 4605 ], [ 1744677450000, 3249 ], [ 1744677480000, 4421 ], [ 1744677510000, 2998 ], [ 1744677540000, 4700 ], [ 1744677570000, 3598 ], [ 1744677600000, 5781 ], [ 1744677630000, 3734 ], [ 1744677660000, 4510 ], [ 1744677690000, 3752 ], [ 1744677720000, 4447 ], [ 1744677750000, 3523 ], [ 1744677780000, 4187 ], [ 1744677810000, 3640 ], [ 1744677840000, 3900 ], [ 1744677870000, 3514 ], [ 1744677900000, 4863 ], [ 1744677930000, 3565 ], [ 1744677960000, 4335 ], [ 1744677990000, 3533 ], [ 1744678020000, 4307 ], [ 1744678050000, 3556 ], [ 1744678080000, 4179 ], [ 1744678110000, 3664 ], [ 1744678140000, 4362 ], [ 1744678170000, 3222 ], [ 1744678200000, 4750 ], [ 1744678230000, 3546 ], [ 1744678260000, 4601 ], [ 1744678290000, 3702 ], [ 1744678320000, 4564 ], [ 1744678350000, 3610 ], [ 1744678380000, 4130 ], [ 1744678410000, 3412 ], [ 1744678440000, 4614 ], [ 1744678470000, 3522 ], [ 1744678500000, 4148 ], [ 1744678530000, 3408 ], [ 1744678560000, 4261 ], [ 1744678590000, 3607 ], [ 1744678620000, 4172 ], [ 1744678650000, 3529 ], [ 1744678680000, 4227 ], [ 1744678710000, 3487 ], [ 1744678740000, 4298 ], [ 1744678770000, 3609 ], [ 1744678800000, 7230 ], [ 1744678830000, 3818 ], [ 1744678860000, 11924 ], [ 1744678890000, 27269 ], [ 1744678920000, 5073 ], [ 1744678950000, 3474 ], [ 1744678980000, 4474 ], [ 1744679010000, 3536 ], [ 1744679040000, 4525 ], [ 1744679070000, 3503 ], [ 1744679100000, 4194 ], [ 1744679130000, 3557 ], [ 1744679160000, 4259 ], [ 1744679190000, 3611 ], [ 1744679220000, 4218 ], [ 1744679250000, 3622 ], [ 1744679280000, 4417 ], [ 1744679310000, 3730 ], [ 1744679340000, 4204 ], [ 1744679370000, 3641 ], [ 1744679400000, 4849 ], [ 1744679430000, 3803 ], [ 1744679460000, 4398 ], [ 1744679490000, 3674 ], [ 1744679520000, 4727 ], [ 1744679550000, 3926 ], [ 1744679580000, 4173 ], [ 1744679610000, 3531 ], [ 1744679640000, 4968 ], [ 1744679670000, 3432 ], [ 1744679700000, 5059 ], [ 1744679730000, 3560 ], [ 1744679760000, 4087 ], [ 1744679790000, 3590 ], [ 1744679820000, 4436 ], [ 1744679850000, 5299 ], [ 1744679880000, 4320 ], [ 1744679910000, 3861 ], [ 1744679940000, 4511 ], [ 1744679970000, 3711 ], [ 1744680000000, 6021 ], [ 1744680030000, 3942 ], [ 1744680060000, 4800 ], [ 1744680090000, 3681 ], [ 1744680120000, 4592 ], [ 1744680150000, 3560 ], [ 1744680180000, 4194 ], [ 1744680210000, 3490 ], [ 1744680240000, 4971 ], [ 1744680270000, 4009 ], [ 1744680300000, 4837 ], [ 1744680330000, 3227 ], [ 1744680360000, 4531 ], [ 1744680390000, 2888 ], [ 1744680420000, 5083 ], [ 1744680450000, 3557 ], [ 1744680480000, 4207 ], [ 1744680510000, 3373 ], [ 1744680540000, 4482 ], [ 1744680570000, 3110 ], [ 1744680600000, 13551 ], [ 1744680630000, 17159 ], [ 1744680660000, 6284 ], [ 1744680690000, 9924 ], [ 1744680720000, 4547 ], [ 1744680750000, 3474 ], [ 1744680780000, 4312 ], [ 1744680810000, 3689 ], [ 1744680840000, 4680 ], [ 1744680870000, 3609 ], [ 1744680900000, 4886 ], [ 1744680930000, 3842 ], [ 1744680960000, 4810 ], [ 1744680990000, 4102 ], [ 1744681020000, 4594 ], [ 1744681050000, 4168 ], [ 1744681080000, 4562 ], [ 1744681110000, 4506 ], [ 1744681140000, 5243 ], [ 1744681170000, 5135 ], [ 1744681200000, 6671 ], [ 1744681230000, 3806 ], [ 1744681260000, 4535 ], [ 1744681290000, 3721 ], [ 1744681320000, 4799 ], [ 1744681350000, 3909 ], [ 1744681380000, 4261 ], [ 1744681410000, 3671 ], [ 1744681440000, 4359 ], [ 1744681470000, 4063 ], [ 1744681500000, 5231 ], [ 1744681530000, 3778 ], [ 1744681560000, 4684 ], [ 1744681590000, 4072 ], [ 1744681620000, 5029 ], [ 1744681650000, 3700 ], [ 1744681680000, 4670 ], [ 1744681710000, 3557 ], [ 1744681740000, 4590 ], [ 1744681770000, 3041 ], [ 1744681800000, 5043 ], [ 1744681830000, 3530 ], [ 1744681860000, 6807 ], [ 1744681890000, 4455 ], [ 1744681920000, 6841 ], [ 1744681950000, 4519 ], [ 1744681980000, 6617 ], [ 1744682010000, 4633 ], [ 1744682040000, 5997 ], [ 1744682070000, 4446 ], [ 1744682100000, 5569 ], [ 1744682130000, 4324 ], [ 1744682160000, 5354 ], [ 1744682190000, 7245 ], [ 1744682220000, 5258 ], [ 1744682250000, 4296 ], [ 1744682280000, 5349 ], [ 1744682310000, 4479 ], [ 1744682340000, 5127 ], [ 1744682370000, 4006 ], [ 1744682400000, 19058 ], [ 1744682430000, 14501 ], [ 1744682460000, 3810 ], [ 1744682490000, 12368 ], [ 1744682520000, 6976 ], [ 1744682550000, 4399 ], [ 1744682580000, 5482 ], [ 1744682610000, 4524 ], [ 1744682640000, 5478 ], [ 1744682670000, 4920 ], [ 1744682700000, 5347 ], [ 1744682730000, 4427 ], [ 1744682760000, 5102 ], [ 1744682790000, 4441 ], [ 1744682820000, 5596 ], [ 1744682850000, 4888 ], [ 1744682880000, 5306 ], [ 1744682910000, 4825 ], [ 1744682940000, 5897 ], [ 1744682970000, 4481 ], [ 1744683000000, 6086 ], [ 1744683030000, 4910 ], [ 1744683060000, 5676 ], [ 1744683090000, 3626 ], [ 1744683120000, 6929 ], [ 1744683150000, 4601 ], [ 1744683180000, 5525 ], [ 1744683210000, 4500 ], [ 1744683240000, 5617 ], [ 1744683270000, 4503 ], [ 1744683300000, 6328 ], [ 1744683330000, 4557 ], [ 1744683360000, 5356 ], [ 1744683390000, 4413 ], [ 1744683420000, 5335 ], [ 1744683450000, 4640 ], [ 1744683480000, 5399 ], [ 1744683510000, 4298 ], [ 1744683540000, 5415 ], [ 1744683570000, 4540 ], [ 1744683600000, 6949 ], [ 1744683630000, 4574 ], [ 1744683660000, 5757 ], [ 1744683690000, 4669 ], [ 1744683720000, 5706 ], [ 1744683750000, 4472 ], [ 1744683780000, 5386 ], [ 1744683810000, 4490 ], [ 1744683840000, 5104 ], [ 1744683870000, 4201 ], [ 1744683900000, 5979 ], [ 1744683930000, 4853 ], [ 1744683960000, 6691 ], [ 1744683990000, 4572 ], [ 1744684020000, 5554 ], [ 1744684050000, 5244 ], [ 1744684080000, 5392 ], [ 1744684110000, 4550 ] ] + } ] +}`, + }, + } { + t.Run(fmt.Sprintf("%s", i), func(t *testing.T) { + metadata.SetUser(ctx, "username:test", spaceUid, "true") + + res, err := queryTsWithPromEngine(ctx, c.queryTs) + assert.Nil(t, err) + excepted, err := json.Marshal(res) + assert.Nil(t, err) + assert.JSONEq(t, c.result, string(excepted)) + }) + } +} + func TestQueryTsWithEs(t *testing.T) { ctx := metadata.InitHashID(context.Background()) + viper.Set(bkapi.BkAPIAddressConfigPath, mock.EsUrlDomain) + spaceUid := influxdb.SpaceUid tableID := influxdb.ResultTableEs @@ -133,7 +260,7 @@ func TestQueryTsWithEs(t *testing.T) { } } -func TestQueryReference(t *testing.T) { +func TestQueryReferenceWithEs(t *testing.T) { ctx := metadata.InitHashID(context.Background()) spaceUid := influxdb.SpaceUid @@ -143,7 +270,6 @@ func TestQueryReference(t *testing.T) { promql.MockEngine() influxdb.MockSpaceRouter(ctx) - mock.Init() defaultStart := time.UnixMilli(1741154079123) // 2025-03-05 13:54:39 defaultEnd := time.UnixMilli(1741155879987) // 2025-03-05 14:24:39 @@ -754,7 +880,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: "1723595000234", }, total: 1e4, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"e058129ae18bff87c95e83f24584e654","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c124dae69af9b86a7128ee4281820158","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"43a36f412886bf30b0746562513638d3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"0826407be7f04f19086774ed68eac8dd","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"d56b4120194eb37f53410780da777d43","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"e058129ae18bff87c95e83f24584e654","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"c124dae69af9b86a7128ee4281820158","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"c7f73abf7e865a4b4d7fc608387d01cf","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"39c3ec662881e44bf26d2a6bfc0e35c3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"58e03ce0b9754bf0657d49a5513adcb5","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"43a36f412886bf30b0746562513638d3","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"218ceafd04f89b39cda7954e51f4a48a","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"8d9abe9b782fe3a1272c93f0af6b39e1","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594211000","dtEventTimeStamp":"1723594211000"},{"__data_label":"bkbase_es","__doc_id":"0826407be7f04f19086774ed68eac8dd","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"},{"__data_label":"bkbase_es","__doc_id":"d56b4120194eb37f53410780da777d43","__ext.io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.bk_base_es","_time":"1723594224000","dtEventTimeStamp":"1723594224000"}]`, }, "query es with multi rt and multi from 0 - 5": { queryTs: &structured.QueryTs{ @@ -783,7 +909,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { From: function.IntPoint(0), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -792,8 +918,8 @@ func TestQueryRawWithInstance(t *testing.T) { }, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5}}`, + expected: `[{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"from":5},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":5}}`, }, "query es with multi rt and multi from 5 - 10": { queryTs: &structured.QueryTs{ @@ -823,7 +949,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, IsMultiFrom: true, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { From: function.IntPoint(5), }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -832,8 +958,8 @@ func TestQueryRawWithInstance(t *testing.T) { }, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":7},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":8}}`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"from":7},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"from":8}}`, }, "query es with multi rt and one from 0 - 5": { queryTs: &structured.QueryTs{ @@ -864,8 +990,8 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]}}`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"}]`, + options: `{"result_table.es|http://127.0.0.1:93002":{"search_after":["1","5"]},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["1","5"]}}`, }, "query es with multi rt and one from 5 - 10": { queryTs: &structured.QueryTs{ @@ -896,8 +1022,8 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 246, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, - options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]},"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]}}`, + expected: `[{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + options: `{"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"search_after":["2","5"]},"result_table.es|http://127.0.0.1:93002":{"search_after":["2","5"]}}`, }, "query_bk_base_es_1 to 1": { queryTs: &structured.QueryTs{ @@ -915,7 +1041,7 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, }, total: 1e4, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, + expected: `[{"__data_label":"es","__doc_id":"4f3a5e9c167097c9658e88b2f32364b2","__ext.container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f","__index":"v2_2_bklog_bk_unify_query_20240814_0","__result_table":"result_table.es","_time":"1723594209000","dtEventTimeStamp":"1723594209000"}]`, }, "query with scroll - 1": { queryTs: &structured.QueryTs{ @@ -946,9 +1072,9 @@ func TestQueryRawWithInstance(t *testing.T) { End: end, Scroll: "5m", }, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"1"},{"__data_label":"es","__doc_id":"1","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"1"},{"__data_label":"bkbase_es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"2"},{"__data_label":"es","__doc_id":"2","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"2"},{"__data_label":"bkbase_es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"3"},{"__data_label":"es","__doc_id":"3","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"3"},{"__data_label":"bkbase_es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"4"},{"__data_label":"es","__doc_id":"4","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"4"},{"__data_label":"bkbase_es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"1","b":"5"},{"__data_label":"es","__doc_id":"5","__index":"result_table_index","__result_table":"result_table.es","a":"1","b":"5"}]`, total: 246, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"}}`, + options: `{"result_table.es|http://127.0.0.1:93002":{"scroll_id":"one"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"one"}}`, }, "query with scroll - 2": { queryTs: &structured.QueryTs{ @@ -978,7 +1104,7 @@ func TestQueryRawWithInstance(t *testing.T) { Start: start, End: end, ResultTableOptions: map[string]*metadata.ResultTableOption{ - "result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es": { + "result_table.es|http://127.0.0.1:93002": { ScrollID: "one", }, "result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es": { @@ -987,9 +1113,9 @@ func TestQueryRawWithInstance(t *testing.T) { }, Scroll: "5m", }, - expected: `[{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"3"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"4"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"bkbase_es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"5"},{"__address":"http://127.0.0.1:12001/bk_data/query_sync/es","__data_label":"es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"5"}]`, + expected: `[{"__data_label":"bkbase_es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"1"},{"__data_label":"es","__doc_id":"6","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"1"},{"__data_label":"bkbase_es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"2"},{"__data_label":"es","__doc_id":"7","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"2"},{"__data_label":"bkbase_es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"3"},{"__data_label":"es","__doc_id":"8","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"3"},{"__data_label":"bkbase_es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"4"},{"__data_label":"es","__doc_id":"9","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"4"},{"__data_label":"bkbase_es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.bk_base_es","a":"2","b":"5"},{"__data_label":"es","__doc_id":"10","__index":"result_table_index","__result_table":"result_table.es","a":"2","b":"5"}]`, total: 246, - options: `{"result_table.es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"}}`, + options: `{"result_table.es|http://127.0.0.1:93002":{"scroll_id":"two"},"result_table.bk_base_es|http://127.0.0.1:12001/bk_data/query_sync/es":{"scroll_id":"two"}}`, }, } diff --git a/pkg/unify-query/tsdb/bksql/client_test.go b/pkg/unify-query/tsdb/bksql/client_test.go index 593bb0a52..6adedb735 100644 --- a/pkg/unify-query/tsdb/bksql/client_test.go +++ b/pkg/unify-query/tsdb/bksql/client_test.go @@ -30,7 +30,7 @@ var ( func MockClient() *bksql.Client { if client == nil { - client = (&bksql.Client{}).WithUrl(mock.BkSQLUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) + client = (&bksql.Client{}).WithUrl(mock.BkBaseUrl).WithCurl(&curl.HttpCurl{Log: log.DefaultLogger}) } return client diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 6e1fde546..7797da397 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -125,9 +125,6 @@ func (f *QueryFactory) FieldMap() map[string]string { func (f *QueryFactory) ReloadListData(data map[string]any) map[string]any { newData := make(map[string]any) - newData[KeyIndex] = f.query.DB - newData[KeyTableID] = f.query.TableID - newData[KeyDataLabel] = f.query.DataLabel fieldMap := f.FieldMap() for k, d := range data { @@ -150,6 +147,134 @@ func (f *QueryFactory) ReloadListData(data map[string]any) map[string]any { return newData } +func (f *QueryFactory) FormatDataToQueryResult(ctx context.Context, list []map[string]interface{}) (*prompb.QueryResult, error) { + res := &prompb.QueryResult{} + + if len(list) == 0 { + return res, nil + } + + encodeFunc := metadata.GetPromDataFormat(ctx).EncodeFunc() + // 获取 metricLabel + metricLabel := f.query.MetricLabels(ctx) + tsMap := map[string]*prompb.TimeSeries{} + + // 先获取维度的 key 保证顺序一致 + keys := make([]string, 0) + for _, d := range list { + // 优先获取时间和值 + var ( + vt int64 + vv float64 + + vtLong interface{} + vvDouble interface{} + + ok bool + ) + + if d == nil { + continue + } + + nd := f.ReloadListData(d) + if len(keys) == 0 { + for k := range nd { + keys = append(keys, k) + } + sort.Strings(keys) + } + + lbl := make([]prompb.Label, 0) + for _, k := range keys { + switch k { + case sqlExpr.TimeStamp: + if _, ok = nd[k]; ok { + vtLong = nd[k] + } + case sqlExpr.Value: + if _, ok = nd[k]; ok { + vvDouble = nd[k] + } + default: + // 获取维度信息 + val, err := getValue(k, nd) + if err != nil { + log.Errorf(ctx, "get dimension (%s) value error in %+v %s", k, d, err.Error()) + continue + } + + if encodeFunc != nil { + k = encodeFunc(k) + } + + lbl = append(lbl, prompb.Label{ + Name: k, + Value: val, + }) + + } + } + + if vtLong == nil { + vtLong = f.start.UnixMilli() + } + + switch vtLong.(type) { + case int64: + vt = vtLong.(int64) + case float64: + vt = int64(vtLong.(float64)) + default: + return res, fmt.Errorf("%s type is error %T, %v", dtEventTimeStamp, vtLong, vtLong) + } + + if vvDouble == nil { + continue + } + switch vvDouble.(type) { + case int64: + vv = float64(vvDouble.(int64)) + case float64: + vv = vvDouble.(float64) + default: + return res, fmt.Errorf("%s type is error %T, %v", sqlExpr.Value, vvDouble, vvDouble) + } + + // 如果是非时间聚合计算，则无需进行指标名的拼接作用 + if metricLabel != nil { + lbl = append(lbl, *metricLabel) + } + + var buf strings.Builder + for _, l := range lbl { + buf.WriteString(l.String()) + } + + // 同一个 series 进行合并分组 + key := buf.String() + if _, ok := tsMap[key]; !ok { + tsMap[key] = &prompb.TimeSeries{ + Labels: lbl, + Samples: make([]prompb.Sample, 0), + } + } + + tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ + Value: vv, + Timestamp: vt, + }) + } + + // 转换结构体 + res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) + for _, ts := range tsMap { + res.Timeseries = append(res.Timeseries, ts) + } + + return res, nil +} + func (f *QueryFactory) getTheDateIndexFilters() (theDateFilter string, err error) { // bkbase 使用 时区东八区 转换为 thedate loc, err := time.LoadLocation("Asia/Shanghai") diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 736e7e591..df457dc54 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -17,7 +17,6 @@ import ( "time" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" @@ -170,122 +169,6 @@ func (i *Instance) dims(dims []string, field string) []string { return dimensions } -func (i *Instance) formatData(ctx context.Context, start time.Time, query *metadata.Query, keys []string, list []map[string]interface{}) (*prompb.QueryResult, error) { - res := &prompb.QueryResult{} - - if len(list) == 0 { - return res, nil - } - // 维度结构体为空则任务异常 - if len(keys) == 0 { - return res, fmt.Errorf("SelectFieldsOrder is empty") - } - - // 获取 metricLabel - metricLabel := query.MetricLabels(ctx) - - tsMap := make(map[string]*prompb.TimeSeries, 0) - for _, d := range list { - // 优先获取时间和值 - var ( - vt int64 - vv float64 - - vtLong interface{} - vvDouble interface{} - - ok bool - ) - - if d == nil { - continue - } - - lbl := make([]prompb.Label, 0) - for _, k := range keys { - switch k { - case sqlExpr.TimeStamp: - if _, ok = d[k]; ok { - vtLong = d[k] - } - case sqlExpr.Value: - if _, ok = d[k]; ok { - vvDouble = d[k] - } - default: - // 获取维度信息 - val, err := getValue(k, d) - if err != nil { - log.Errorf(ctx, "get dimension (%s) value error in %+v %s", k, d, err.Error()) - continue - } - - lbl = append(lbl, prompb.Label{ - Name: k, - Value: val, - }) - } - } - - if vtLong == nil { - vtLong = start.UnixMilli() - } - - switch vtLong.(type) { - case int64: - vt = vtLong.(int64) - case float64: - vt = int64(vtLong.(float64)) - default: - return res, fmt.Errorf("%s type is error %T, %v", dtEventTimeStamp, vtLong, vtLong) - } - - if vvDouble == nil { - continue - } - switch vvDouble.(type) { - case int64: - vv = float64(vvDouble.(int64)) - case float64: - vv = vvDouble.(float64) - default: - return res, fmt.Errorf("%s type is error %T, %v", sqlExpr.Value, vvDouble, vvDouble) - } - - // 如果是非时间聚合计算，则无需进行指标名的拼接作用 - if metricLabel != nil { - lbl = append(lbl, *metricLabel) - } - - var buf strings.Builder - for _, l := range lbl { - buf.WriteString(l.String()) - } - - // 同一个 series 进行合并分组 - key := buf.String() - if _, ok := tsMap[key]; !ok { - tsMap[key] = &prompb.TimeSeries{ - Labels: lbl, - Samples: make([]prompb.Sample, 0), - } - } - - tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ - Value: vv, - Timestamp: vt, - }) - } - - // 转换结构体 - res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) - for _, ts := range tsMap { - res.Timeseries = append(res.Timeseries, ts) - } - - return res, nil -} - func (i *Instance) getFieldsMap(ctx context.Context, sql string) (map[string]string, error) { fieldsMap := make(map[string]string) @@ -427,7 +310,11 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star } for _, list := range data.List { - dataCh <- queryFactory.ReloadListData(list) + newData := queryFactory.ReloadListData(list) + newData[KeyIndex] = query.DB + newData[KeyTableID] = query.TableID + newData[KeyDataLabel] = query.DataLabel + dataCh <- newData } total = int64(data.TotalRecordSize) @@ -484,7 +371,7 @@ func (i *Instance) QuerySeriesSet(ctx context.Context, query *metadata.Query, st return storage.ErrSeriesSet(fmt.Errorf("记录数(%d)超过限制(%d)", data.TotalRecords, i.maxLimit)) } - qr, err := i.formatData(ctx, start, query, data.SelectFieldsOrder, data.List) + qr, err := queryFactory.FormatDataToQueryResult(ctx, data.List) if err != nil { return storage.ErrSeriesSet(err) } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index adfe34b30..4771bd553 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -74,7 +74,7 @@ func TestInstance_QuerySeriesSet(t *testing.T) { mock.BkSQL.Set(map[string]any{ // doris - "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":20,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":9,"bkbase_apigw_api":29},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c1","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"c2","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"varchar(65533)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":7,"connect_db":66,"match_query_routing_rule":0,"check_permission":72,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":13168,"timetaken":0.152,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, @@ -1162,7 +1162,7 @@ func createTestInstance(ctx context.Context) *bksql.Instance { mock.Init() ins, err := bksql.NewInstance(ctx, &bksql.Options{ - Address: mock.BkSQLUrl, + Address: mock.BkBaseUrl, Timeout: time.Minute, MaxLimit: 1e4, Tolerance: 5, diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 2b9b12dea..27cb55b61 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -87,8 +87,12 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, var ( window time.Duration timezone string - dimensionMap = make(map[string]struct{}) + dimensionMap = map[string]struct{}{ + FieldValue: {}, + FieldTime: {}, + } ) + for _, agg := range aggregates { for _, dim := range agg.Dimensions { var ( @@ -183,7 +187,7 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, var orderField string switch order.Name { case FieldValue: - orderField = d.valueField + orderField = Value case FieldTime: orderField = TimeStamp default: @@ -259,63 +263,77 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) switch c.Operator { // 处理等于类操作符（=, IN, LIKE） case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: + if len(c.Value) == 1 && c.Value[0] == "" { + op = "IS" + val = "NULL" + break + } + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { op = "IN" val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + break + } + + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "LIKE" } else { - var format string - if c.IsWildcard { - format = "'%%%s%%'" - op = "LIKE" + format = "'%s'" + if d.checkMatchALL(c.DimensionName) { + op = "MATCH_PHRASE_PREFIX" } else { - format = "'%s'" - if d.checkMatchALL(c.DimensionName) { - op = "MATCH_PHRASE_PREFIX" - } else { - op = "=" - } + op = "=" } + } - var filter []string - for _, v := range c.Value { - filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) - } - key = "" - if len(filter) == 1 { - val = filter[0] - } else { - val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) - } + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " OR ")) } // 处理不等于类操作符（!=, NOT IN, NOT LIKE） case metadata.ConditionNotEqual, metadata.ConditionNotContains: + if len(c.Value) == 1 && c.Value[0] == "" { + op = "IS NOT" + val = "NULL" + break + } + if len(c.Value) > 1 && !c.IsWildcard && !d.checkMatchALL(c.DimensionName) { op = "NOT IN" val = fmt.Sprintf("('%s')", strings.Join(c.Value, "', '")) + break + } + + var format string + if c.IsWildcard { + format = "'%%%s%%'" + op = "NOT LIKE" } else { - var format string - if c.IsWildcard { - format = "'%%%s%%'" - op = "NOT LIKE" + format = "'%s'" + if d.checkMatchALL(c.DimensionName) { + op = "NOT MATCH_PHRASE_PREFIX" } else { - format = "'%s'" - if d.checkMatchALL(c.DimensionName) { - op = "NOT MATCH_PHRASE_PREFIX" - } else { - op = "!=" - } + op = "!=" } + } - var filter []string - for _, v := range c.Value { - filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) - } - key = "" - if len(filter) == 1 { - val = filter[0] - } else { - val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) - } + var filter []string + for _, v := range c.Value { + filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) + } + key = "" + if len(filter) == 1 { + val = filter[0] + } else { + val = fmt.Sprintf("(%s)", strings.Join(filter, " AND ")) } // 处理正则表达式匹配 case metadata.ConditionRegEqual: diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index a20636c68..6738685d9 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -184,6 +184,10 @@ func (i *Instance) getMappings(ctx context.Context, conn Connect, aliases []stri return nil, err } mappingMap, err := client.GetMapping().Index(aliases...).Type("").Do(ctx) + if err != nil { + log.Warnf(ctx, "get mapping error: %s", err.Error()) + return nil, err + } indexes := make([]string, 0, len(mappingMap)) for index := range mappingMap { @@ -808,16 +812,16 @@ func (i *Instance) QuerySeriesSet( conn: conn, } - mappings, err1 := i.getMappings(ctx, qo.conn, qo.indexes) + mappings, errMapping := i.getMappings(ctx, qo.conn, qo.indexes) // index 不存在，mappings 获取异常直接返回空 if len(mappings) == 0 { - log.Warnf(ctx, "index is empty with %v", qo.indexes) + log.Warnf(ctx, "index is empty with %v with %s error %v", qo.indexes, qo.conn.String(), errMapping) return } - if err1 != nil { + if errMapping != nil { rets <- &TimeSeriesResult{ - Error: err1, + Error: errMapping, } return } diff --git a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go index 61da29e76..b07efb67c 100644 --- a/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go +++ b/pkg/unify-query/tsdb/victoriaMetrics/instance_test.go @@ -31,7 +31,7 @@ var ( vmRt string = "2_bcs_prom_computation_result_table_00000" instance = &Instance{ - url: mock.VmUrl, + url: mock.BkBaseUrl, timeout: time.Minute * 5, curl: &curl.HttpCurl{}, } @@ -103,7 +103,7 @@ func TestInstance_DirectQueryRange(t *testing.T) { }) instance := &Instance{ - url: mock.VmUrl, + url: mock.BkBaseUrl, timeout: time.Minute * 5, curl: &curl.HttpCurl{}, } From ef1aa286329b76faea8dc22156d02a3bbe309011 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 16 Apr 2025 18:14:36 +0800 Subject: [PATCH 084/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/query_string.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/query_string.go b/pkg/unify-query/tsdb/elasticsearch/query_string.go index 450805bef..f39b424aa 100644 --- a/pkg/unify-query/tsdb/elasticsearch/query_string.go +++ b/pkg/unify-query/tsdb/elasticsearch/query_string.go @@ -53,7 +53,7 @@ func (s *QueryString) ToDSL() (elastic.Query, error) { } }() - if s.q == "" { + if s.q == "" || s.q == "*" { return nil, nil } From 910e7a90c2b6bc51b39c6cfe326c52b828cbddf6 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 16 Apr 2025 18:37:24 +0800 Subject: [PATCH 085/104] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=20tagvalues?= =?UTF-8?q?=20=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3=EF=BC=8C=E8=B6=85?= =?UTF-8?q?=E8=BF=87=E4=B8=80=E5=A4=A9=E7=9A=84=E4=BD=BF=E7=94=A8=20labelv?= =?UTF-8?q?alues=20=E6=9F=A5=E8=AF=A2=20#1010158081123432529?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/api.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/unify-query/service/http/api.go b/pkg/unify-query/service/http/api.go index da70c1dab..d3ed168f6 100644 --- a/pkg/unify-query/service/http/api.go +++ b/pkg/unify-query/service/http/api.go @@ -271,7 +271,6 @@ func HandlerTagValues(c *gin.Context) { span.Set("direct-label-values-matcher", matcher.String()) span.Set("direct-label-values-size", qry.Size) - span.Set("direct-label-values-size", vmExpand) res, err = instance.DirectLabelValues(ctx, name, start, end, qry.Size, matcher) } else { From a708cd9017565dbeceb5e4e1960121b9021c4b21 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 16 Apr 2025 19:48:07 +0800 Subject: [PATCH 086/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index b1d46cbe3..e82a4dd11 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -351,6 +351,7 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot size, options, queryErr := instance.QueryRawData(ctx, qry, start, end, dataCh) if queryErr != nil { message.WriteString(fmt.Sprintf("query %s:%s is error: %s ", qry.TableID, qry.Fields, queryErr.Error())) + return } // 如果配置了 IsMultiFrom，则无需使用 scroll 和 searchAfter 配置 From 47d1ab3fd90ad77e410a39dfd5ac4a4af152c491 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 17 Apr 2025 11:20:17 +0800 Subject: [PATCH 087/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query.go | 2 ++ pkg/unify-query/tsdb/bksql/instance.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index e82a4dd11..203c30232 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -179,6 +179,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot lock sync.Mutex ) + list = make([]map[string]any, 0) + // 构建查询路由列表 if queryTs.SpaceUid == "" { queryTs.SpaceUid = metadata.GetUser(ctx).SpaceUid diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index df457dc54..29b020c76 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -261,6 +261,7 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star span.Set("query-raw-end", end) if start.UnixMilli() > end.UnixMilli() || start.UnixMilli() == 0 { + err = fmt.Errorf("start time must less than end time") return } From 2cc8db1fcdbc3e87e2878917f4bf3100d4415143 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 17 Apr 2025 15:23:44 +0800 Subject: [PATCH 088/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance.go | 4 ---- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 2 -- 2 files changed, 6 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 29b020c76..6f368b629 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -13,7 +13,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "time" "github.com/prometheus/prometheus/model/labels" @@ -197,9 +196,6 @@ func (i *Instance) getFieldsMap(ctx context.Context, sql string) (map[string]str continue } - // 忽略大小写，所以统一转成小写 - k = strings.ToLower(k) - fieldsMap[k] = v } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 27cb55b61..e3560fe9a 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -379,8 +379,6 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) func (d *DorisSQLExpr) checkMatchALL(k string) bool { if d.fieldsMap != nil { - // 忽略大小写，所以统一转成小写 - k = strings.ToLower(k) if t, ok := d.fieldsMap[k]; ok { if t == DorisTypeText { return true From bd437994ad027a6d6e7b6004c72c49d0fc89a17c Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 18 Apr 2025 11:32:19 +0800 Subject: [PATCH 089/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 203c30232..439a31002 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -439,6 +439,9 @@ func queryReferenceWithPromEngine(ctx context.Context, queryTs *structured.Query if err != nil { return nil, err } + } else { + // reference 接口背后都使用了存储引擎计算，所以在不特殊指定的情况下，使用 1s 补点逻辑，防止出的数据异常 + lookBackDelta = time.Second } instance := prometheus.NewInstance(ctx, promql.GlobalEngine, &prometheus.QueryRangeStorage{ From ba43ca1e5a259f31f444d75971b8ff0e5bf45183 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 18 Apr 2025 12:01:05 +0800 Subject: [PATCH 090/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index 17b28e403..ba70a5df6 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -200,6 +200,20 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, want: "`env` IN ('prod', 'test')", }, + { + name: "test IN operator with wildcard", + condition: metadata.AllConditions{ + { + { + DimensionName: "env", + Value: []string{"prod", "test"}, + Operator: metadata.ConditionContains, + IsWildcard: true, + }, + }, + }, + want: "(`env` LIKE '%prod%' OR `env` LIKE '%test%')", + }, { name: "doris test empty value", condition: metadata.AllConditions{ From 062ed0eb0016c0931afd0d9a18c18076e78ae30e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 18 Apr 2025 12:12:34 +0800 Subject: [PATCH 091/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 7797da397..45ce8f311 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -375,12 +375,12 @@ func (f *QueryFactory) SQL() (sql string, err error) { sort.Strings(orderFields) sql += " ORDER BY " + strings.Join(orderFields, ", ") } - if f.query.From > 0 { - sql += fmt.Sprintf(" OFFSET %d", f.query.From) - } if f.query.Size > 0 { sql += fmt.Sprintf(" LIMIT %d", f.query.Size) } + if f.query.From > 0 { + sql += fmt.Sprintf(" OFFSET %d", f.query.From) + } span.Set("sql", sql) return From 62e35e88d32ad71c11f483f964acb2b507c79cc1 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 18 Apr 2025 14:37:14 +0800 Subject: [PATCH 092/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index e3560fe9a..acb34d16f 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -483,7 +483,7 @@ func (d *DorisSQLExpr) dimTransform(s string) (string, bool) { fs := strings.Split(s, ".") if len(fs) > 1 { - return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], "][")), true + return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)), true } return fmt.Sprintf("`%s`", s), false } From 91e5bad43923d3e22e09fa6cccdf73fac72db1f8 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 18 Apr 2025 15:01:36 +0800 Subject: [PATCH 093/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/service/http/query_test.go | 2 +- pkg/unify-query/tsdb/bksql/format_test.go | 2 +- pkg/unify-query/tsdb/bksql/instance_test.go | 42 +++++++++++++++++++-- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 3 +- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index cac4729ae..73ad7ceef 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -59,7 +59,7 @@ func TestQueryTsWithDoris(t *testing.T) { "SELECT *, `gseIndex` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662180000 AND `dtEventTimeStamp` <= 1744684113000 AND `thedate` = '20250415' ORDER BY `_value_` ASC LIMIT 1": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":12,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250415,"dteventtimestamp":1744662180000,"dteventtime":null,"localtime":null,"__shard_key__":29077703953,"__ext":"{\"container_id\":\"375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2\",\"container_image\":\"sha256:3a0506f06f1467e93c3a582203aac1a7501e77091572ec9612ddeee4a4dbbdb8\",\"container_name\":\"unify-query\",\"io_kubernetes_pod\":\"bk-datalink-unify-query-6df8bcc4c9-rk4sc\",\"io_kubernetes_pod_ip\":\"127.0.0.1\",\"io_kubernetes_pod_namespace\":\"blueking\",\"io_kubernetes_pod_uid\":\"558c5b17-b221-47e1-aa66-036cc9b43e2a\",\"io_kubernetes_workload_name\":\"bk-datalink-unify-query-6df8bcc4c9\",\"io_kubernetes_workload_type\":\"ReplicaSet\"}","cloudid":0.0,"file":"http/handler.go:320","gseindex":2450131.0,"iterationindex":19.0,"level":"info","log":"2025-04-14T20:22:59.982Z\tinfo\thttp/handler.go:320\t[5108397435e997364f8dc1251533e65e] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[strategy:9155] Connection:[keep-alive] Content-Length:[863] Content-Type:[application/json] Traceparent:[00-5108397435e997364f8dc1251533e65e-ca18e72c0f0eafd4-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"space_uid\":\"bkcc__2\",\"query_list\":[{\"field_name\":\"bscp_config_consume_total_file_change_count\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"app\",\"biz\",\"clientType\"]}],\"time_aggregation\":{\"function\":\"increase\",\"window\":\"1m\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"app\",\"biz\",\"clientType\"],\"conditions\":{\"field_list\":[{\"field_name\":\"releaseChangeStatus\",\"value\":[\"Failed\"],\"op\":\"contains\"},{\"field_name\":\"bcs_cluster_id\",\"value\":[\"BCS-K8S-00000\"],\"op\":\"contains\"}],\"condition_list\":[\"and\"]},\"keep_columns\":[\"_time\",\"a\",\"app\",\"biz\",\"clientType\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1744660260\",\"end_time\":\"1744662120\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2/375597ee636fd5d53cb7b0958823d9ba6534bd24cd698e485c41ca2f01b78ed2-json.log","report_time":"2025-04-14T20:22:59.982Z","serverip":"30.189.37.111","time":"1744662180000","trace_id":"5108397435e997364f8dc1251533e65e","_value_":2450131.0,"_timestamp_":1744662180000}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":182,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":56,"match_query_routing_rule":0,"check_permission":13,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["thedate","dteventtimestamp","dteventtime","localtime","__shard_key__","__ext","cloudid","file","gseindex","iterationindex","level","log","message","path","report_time","serverip","time","trace_id","_value_","_timestamp_"],"total_record_size":8856,"timetaken":0.255,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"dteventtimestamp","field_index":1},{"field_type":"string","field_name":"__c2","field_alias":"dteventtime","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"localtime","field_index":3},{"field_type":"long","field_name":"__c4","field_alias":"__shard_key__","field_index":4},{"field_type":"string","field_name":"__c5","field_alias":"__ext","field_index":5},{"field_type":"double","field_name":"__c6","field_alias":"cloudid","field_index":6},{"field_type":"string","field_name":"__c7","field_alias":"file","field_index":7},{"field_type":"double","field_name":"__c8","field_alias":"gseindex","field_index":8},{"field_type":"double","field_name":"__c9","field_alias":"iterationindex","field_index":9},{"field_type":"string","field_name":"__c10","field_alias":"level","field_index":10},{"field_type":"string","field_name":"__c11","field_alias":"log","field_index":11},{"field_type":"string","field_name":"__c12","field_alias":"message","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"path","field_index":13},{"field_type":"string","field_name":"__c14","field_alias":"report_time","field_index":14},{"field_type":"string","field_name":"__c15","field_alias":"serverip","field_index":15},{"field_type":"string","field_name":"__c16","field_alias":"time","field_index":16},{"field_type":"string","field_name":"__c17","field_alias":"trace_id","field_index":17},{"field_type":"double","field_name":"__c18","field_alias":"_value_","field_index":18},{"field_type":"long","field_name":"__c19","field_alias":"_timestamp_","field_index":19}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, // 根据维度 __ext.container_name 进行 count 聚合，同时用值正向排序 - "SELECT CAST(__ext[\"container_name\"] AS STRING) AS `__ext__bk_46__container_name`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 30000 AS INT) * 30000 AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662509999 AND `dtEventTimeStamp` <= 1744684142999 AND `thedate` = '20250415' GROUP BY CAST(__ext[\"container_name\"] AS STRING), _timestamp_ ORDER BY `_timestamp_` ASC, `_value_` ASC LIMIT 2000005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":722,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":6,"bkbase_apigw_api":28},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"__ext__bk_46__container_name":"unify-query","_value_":3684,"_timestamp_":1744662510000},{"__ext__bk_46__container_name":"unify-query","_value_":4012,"_timestamp_":1744662540000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744662570000},{"__ext__bk_46__container_name":"unify-query","_value_":17092,"_timestamp_":1744662600000},{"__ext__bk_46__container_name":"unify-query","_value_":12881,"_timestamp_":1744662630000},{"__ext__bk_46__container_name":"unify-query","_value_":5902,"_timestamp_":1744662660000},{"__ext__bk_46__container_name":"unify-query","_value_":10443,"_timestamp_":1744662690000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744662720000},{"__ext__bk_46__container_name":"unify-query","_value_":3357,"_timestamp_":1744662750000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744662780000},{"__ext__bk_46__container_name":"unify-query","_value_":3683,"_timestamp_":1744662810000},{"__ext__bk_46__container_name":"unify-query","_value_":4353,"_timestamp_":1744662840000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744662870000},{"__ext__bk_46__container_name":"unify-query","_value_":4251,"_timestamp_":1744662900000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744662930000},{"__ext__bk_46__container_name":"unify-query","_value_":4036,"_timestamp_":1744662960000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744662990000},{"__ext__bk_46__container_name":"unify-query","_value_":4351,"_timestamp_":1744663020000},{"__ext__bk_46__container_name":"unify-query","_value_":3651,"_timestamp_":1744663050000},{"__ext__bk_46__container_name":"unify-query","_value_":4096,"_timestamp_":1744663080000},{"__ext__bk_46__container_name":"unify-query","_value_":3618,"_timestamp_":1744663110000},{"__ext__bk_46__container_name":"unify-query","_value_":4100,"_timestamp_":1744663140000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744663170000},{"__ext__bk_46__container_name":"unify-query","_value_":6044,"_timestamp_":1744663200000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744663230000},{"__ext__bk_46__container_name":"unify-query","_value_":4461,"_timestamp_":1744663260000},{"__ext__bk_46__container_name":"unify-query","_value_":3783,"_timestamp_":1744663290000},{"__ext__bk_46__container_name":"unify-query","_value_":4559,"_timestamp_":1744663320000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744663350000},{"__ext__bk_46__container_name":"unify-query","_value_":3869,"_timestamp_":1744663380000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744663410000},{"__ext__bk_46__container_name":"unify-query","_value_":4473,"_timestamp_":1744663440000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744663470000},{"__ext__bk_46__container_name":"unify-query","_value_":4923,"_timestamp_":1744663500000},{"__ext__bk_46__container_name":"unify-query","_value_":3379,"_timestamp_":1744663530000},{"__ext__bk_46__container_name":"unify-query","_value_":4489,"_timestamp_":1744663560000},{"__ext__bk_46__container_name":"unify-query","_value_":3411,"_timestamp_":1744663590000},{"__ext__bk_46__container_name":"unify-query","_value_":4374,"_timestamp_":1744663620000},{"__ext__bk_46__container_name":"unify-query","_value_":3370,"_timestamp_":1744663650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744663680000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744663710000},{"__ext__bk_46__container_name":"unify-query","_value_":4318,"_timestamp_":1744663740000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744663770000},{"__ext__bk_46__container_name":"unify-query","_value_":4334,"_timestamp_":1744663800000},{"__ext__bk_46__container_name":"unify-query","_value_":3767,"_timestamp_":1744663830000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744663860000},{"__ext__bk_46__container_name":"unify-query","_value_":3703,"_timestamp_":1744663890000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744663920000},{"__ext__bk_46__container_name":"unify-query","_value_":3667,"_timestamp_":1744663950000},{"__ext__bk_46__container_name":"unify-query","_value_":3998,"_timestamp_":1744663980000},{"__ext__bk_46__container_name":"unify-query","_value_":3579,"_timestamp_":1744664010000},{"__ext__bk_46__container_name":"unify-query","_value_":4156,"_timestamp_":1744664040000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744664070000},{"__ext__bk_46__container_name":"unify-query","_value_":4344,"_timestamp_":1744664100000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744664130000},{"__ext__bk_46__container_name":"unify-query","_value_":4161,"_timestamp_":1744664160000},{"__ext__bk_46__container_name":"unify-query","_value_":3484,"_timestamp_":1744664190000},{"__ext__bk_46__container_name":"unify-query","_value_":4273,"_timestamp_":1744664220000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744664250000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744664280000},{"__ext__bk_46__container_name":"unify-query","_value_":3619,"_timestamp_":1744664310000},{"__ext__bk_46__container_name":"unify-query","_value_":4013,"_timestamp_":1744664340000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744664370000},{"__ext__bk_46__container_name":"unify-query","_value_":18144,"_timestamp_":1744664400000},{"__ext__bk_46__container_name":"unify-query","_value_":13615,"_timestamp_":1744664430000},{"__ext__bk_46__container_name":"unify-query","_value_":3178,"_timestamp_":1744664460000},{"__ext__bk_46__container_name":"unify-query","_value_":13044,"_timestamp_":1744664490000},{"__ext__bk_46__container_name":"unify-query","_value_":4767,"_timestamp_":1744664520000},{"__ext__bk_46__container_name":"unify-query","_value_":3528,"_timestamp_":1744664550000},{"__ext__bk_46__container_name":"unify-query","_value_":4316,"_timestamp_":1744664580000},{"__ext__bk_46__container_name":"unify-query","_value_":3317,"_timestamp_":1744664610000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744664640000},{"__ext__bk_46__container_name":"unify-query","_value_":3599,"_timestamp_":1744664670000},{"__ext__bk_46__container_name":"unify-query","_value_":4149,"_timestamp_":1744664700000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744664730000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744664760000},{"__ext__bk_46__container_name":"unify-query","_value_":3384,"_timestamp_":1744664790000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744664820000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744664850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744664880000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744664910000},{"__ext__bk_46__container_name":"unify-query","_value_":4244,"_timestamp_":1744664940000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744664970000},{"__ext__bk_46__container_name":"unify-query","_value_":4305,"_timestamp_":1744665000000},{"__ext__bk_46__container_name":"unify-query","_value_":3771,"_timestamp_":1744665030000},{"__ext__bk_46__container_name":"unify-query","_value_":4485,"_timestamp_":1744665060000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744665090000},{"__ext__bk_46__container_name":"unify-query","_value_":4423,"_timestamp_":1744665120000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665150000},{"__ext__bk_46__container_name":"unify-query","_value_":4125,"_timestamp_":1744665180000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744665210000},{"__ext__bk_46__container_name":"unify-query","_value_":4252,"_timestamp_":1744665240000},{"__ext__bk_46__container_name":"unify-query","_value_":3427,"_timestamp_":1744665270000},{"__ext__bk_46__container_name":"unify-query","_value_":5089,"_timestamp_":1744665300000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744665330000},{"__ext__bk_46__container_name":"unify-query","_value_":4349,"_timestamp_":1744665360000},{"__ext__bk_46__container_name":"unify-query","_value_":3188,"_timestamp_":1744665390000},{"__ext__bk_46__container_name":"unify-query","_value_":4556,"_timestamp_":1744665420000},{"__ext__bk_46__container_name":"unify-query","_value_":3372,"_timestamp_":1744665450000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744665480000},{"__ext__bk_46__container_name":"unify-query","_value_":3445,"_timestamp_":1744665510000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744665540000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744665570000},{"__ext__bk_46__container_name":"unify-query","_value_":6235,"_timestamp_":1744665600000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744665630000},{"__ext__bk_46__container_name":"unify-query","_value_":4577,"_timestamp_":1744665660000},{"__ext__bk_46__container_name":"unify-query","_value_":3719,"_timestamp_":1744665690000},{"__ext__bk_46__container_name":"unify-query","_value_":4548,"_timestamp_":1744665720000},{"__ext__bk_46__container_name":"unify-query","_value_":3420,"_timestamp_":1744665750000},{"__ext__bk_46__container_name":"unify-query","_value_":4246,"_timestamp_":1744665780000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744665810000},{"__ext__bk_46__container_name":"unify-query","_value_":4332,"_timestamp_":1744665840000},{"__ext__bk_46__container_name":"unify-query","_value_":3422,"_timestamp_":1744665870000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744665900000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665930000},{"__ext__bk_46__container_name":"unify-query","_value_":4119,"_timestamp_":1744665960000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744665990000},{"__ext__bk_46__container_name":"unify-query","_value_":4144,"_timestamp_":1744666020000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744666050000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744666080000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744666110000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744666140000},{"__ext__bk_46__container_name":"unify-query","_value_":3315,"_timestamp_":1744666170000},{"__ext__bk_46__container_name":"unify-query","_value_":16721,"_timestamp_":1744666200000},{"__ext__bk_46__container_name":"unify-query","_value_":13631,"_timestamp_":1744666230000},{"__ext__bk_46__container_name":"unify-query","_value_":2982,"_timestamp_":1744666260000},{"__ext__bk_46__container_name":"unify-query","_value_":11858,"_timestamp_":1744666290000},{"__ext__bk_46__container_name":"unify-query","_value_":5515,"_timestamp_":1744666320000},{"__ext__bk_46__container_name":"unify-query","_value_":2869,"_timestamp_":1744666350000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744666380000},{"__ext__bk_46__container_name":"unify-query","_value_":3603,"_timestamp_":1744666410000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744666440000},{"__ext__bk_46__container_name":"unify-query","_value_":3264,"_timestamp_":1744666470000},{"__ext__bk_46__container_name":"unify-query","_value_":4377,"_timestamp_":1744666500000},{"__ext__bk_46__container_name":"unify-query","_value_":3443,"_timestamp_":1744666530000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744666560000},{"__ext__bk_46__container_name":"unify-query","_value_":3459,"_timestamp_":1744666590000},{"__ext__bk_46__container_name":"unify-query","_value_":4342,"_timestamp_":1744666620000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744666650000},{"__ext__bk_46__container_name":"unify-query","_value_":4052,"_timestamp_":1744666680000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744666710000},{"__ext__bk_46__container_name":"unify-query","_value_":4128,"_timestamp_":1744666740000},{"__ext__bk_46__container_name":"unify-query","_value_":3499,"_timestamp_":1744666770000},{"__ext__bk_46__container_name":"unify-query","_value_":6209,"_timestamp_":1744666800000},{"__ext__bk_46__container_name":"unify-query","_value_":3575,"_timestamp_":1744666830000},{"__ext__bk_46__container_name":"unify-query","_value_":4543,"_timestamp_":1744666860000},{"__ext__bk_46__container_name":"unify-query","_value_":3604,"_timestamp_":1744666890000},{"__ext__bk_46__container_name":"unify-query","_value_":4579,"_timestamp_":1744666920000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744666950000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744666980000},{"__ext__bk_46__container_name":"unify-query","_value_":3416,"_timestamp_":1744667010000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667040000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744667070000},{"__ext__bk_46__container_name":"unify-query","_value_":5054,"_timestamp_":1744667100000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744667130000},{"__ext__bk_46__container_name":"unify-query","_value_":4313,"_timestamp_":1744667160000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744667190000},{"__ext__bk_46__container_name":"unify-query","_value_":4118,"_timestamp_":1744667220000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744667250000},{"__ext__bk_46__container_name":"unify-query","_value_":4280,"_timestamp_":1744667280000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744667310000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744667340000},{"__ext__bk_46__container_name":"unify-query","_value_":3807,"_timestamp_":1744667370000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744667400000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744667430000},{"__ext__bk_46__container_name":"unify-query","_value_":4630,"_timestamp_":1744667460000},{"__ext__bk_46__container_name":"unify-query","_value_":3845,"_timestamp_":1744667490000},{"__ext__bk_46__container_name":"unify-query","_value_":4361,"_timestamp_":1744667520000},{"__ext__bk_46__container_name":"unify-query","_value_":3572,"_timestamp_":1744667550000},{"__ext__bk_46__container_name":"unify-query","_value_":4095,"_timestamp_":1744667580000},{"__ext__bk_46__container_name":"unify-query","_value_":3535,"_timestamp_":1744667610000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744667640000},{"__ext__bk_46__container_name":"unify-query","_value_":3390,"_timestamp_":1744667670000},{"__ext__bk_46__container_name":"unify-query","_value_":4262,"_timestamp_":1744667700000},{"__ext__bk_46__container_name":"unify-query","_value_":3398,"_timestamp_":1744667730000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667760000},{"__ext__bk_46__container_name":"unify-query","_value_":3429,"_timestamp_":1744667790000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744667820000},{"__ext__bk_46__container_name":"unify-query","_value_":3482,"_timestamp_":1744667850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744667880000},{"__ext__bk_46__container_name":"unify-query","_value_":3612,"_timestamp_":1744667910000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744667940000},{"__ext__bk_46__container_name":"unify-query","_value_":3423,"_timestamp_":1744667970000},{"__ext__bk_46__container_name":"unify-query","_value_":18203,"_timestamp_":1744668000000},{"__ext__bk_46__container_name":"unify-query","_value_":13685,"_timestamp_":1744668030000},{"__ext__bk_46__container_name":"unify-query","_value_":3281,"_timestamp_":1744668060000},{"__ext__bk_46__container_name":"unify-query","_value_":12556,"_timestamp_":1744668090000},{"__ext__bk_46__container_name":"unify-query","_value_":4893,"_timestamp_":1744668120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744668150000},{"__ext__bk_46__container_name":"unify-query","_value_":4336,"_timestamp_":1744668180000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744668210000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744668240000},{"__ext__bk_46__container_name":"unify-query","_value_":3669,"_timestamp_":1744668270000},{"__ext__bk_46__container_name":"unify-query","_value_":3997,"_timestamp_":1744668300000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744668330000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744668360000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744668390000},{"__ext__bk_46__container_name":"unify-query","_value_":3877,"_timestamp_":1744668420000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744668450000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744668480000},{"__ext__bk_46__container_name":"unify-query","_value_":3469,"_timestamp_":1744668510000},{"__ext__bk_46__container_name":"unify-query","_value_":4243,"_timestamp_":1744668540000},{"__ext__bk_46__container_name":"unify-query","_value_":3304,"_timestamp_":1744668570000},{"__ext__bk_46__container_name":"unify-query","_value_":4690,"_timestamp_":1744668600000},{"__ext__bk_46__container_name":"unify-query","_value_":3717,"_timestamp_":1744668630000},{"__ext__bk_46__container_name":"unify-query","_value_":4618,"_timestamp_":1744668660000},{"__ext__bk_46__container_name":"unify-query","_value_":3732,"_timestamp_":1744668690000},{"__ext__bk_46__container_name":"unify-query","_value_":4477,"_timestamp_":1744668720000},{"__ext__bk_46__container_name":"unify-query","_value_":3615,"_timestamp_":1744668750000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744668780000},{"__ext__bk_46__container_name":"unify-query","_value_":3367,"_timestamp_":1744668810000},{"__ext__bk_46__container_name":"unify-query","_value_":4193,"_timestamp_":1744668840000},{"__ext__bk_46__container_name":"unify-query","_value_":3592,"_timestamp_":1744668870000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744668900000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744668930000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744668960000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744668990000},{"__ext__bk_46__container_name":"unify-query","_value_":4375,"_timestamp_":1744669020000},{"__ext__bk_46__container_name":"unify-query","_value_":3386,"_timestamp_":1744669050000},{"__ext__bk_46__container_name":"unify-query","_value_":4281,"_timestamp_":1744669080000},{"__ext__bk_46__container_name":"unify-query","_value_":3410,"_timestamp_":1744669110000},{"__ext__bk_46__container_name":"unify-query","_value_":4545,"_timestamp_":1744669140000},{"__ext__bk_46__container_name":"unify-query","_value_":3724,"_timestamp_":1744669170000},{"__ext__bk_46__container_name":"unify-query","_value_":5903,"_timestamp_":1744669200000},{"__ext__bk_46__container_name":"unify-query","_value_":3672,"_timestamp_":1744669230000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744669260000},{"__ext__bk_46__container_name":"unify-query","_value_":3792,"_timestamp_":1744669290000},{"__ext__bk_46__container_name":"unify-query","_value_":4422,"_timestamp_":1744669320000},{"__ext__bk_46__container_name":"unify-query","_value_":3718,"_timestamp_":1744669350000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744669380000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744669410000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744669440000},{"__ext__bk_46__container_name":"unify-query","_value_":3542,"_timestamp_":1744669470000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744669500000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669530000},{"__ext__bk_46__container_name":"unify-query","_value_":4354,"_timestamp_":1744669560000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669590000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744669620000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744669650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744669680000},{"__ext__bk_46__container_name":"unify-query","_value_":3512,"_timestamp_":1744669710000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744669740000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744669770000},{"__ext__bk_46__container_name":"unify-query","_value_":12171,"_timestamp_":1744669800000},{"__ext__bk_46__container_name":"unify-query","_value_":18129,"_timestamp_":1744669830000},{"__ext__bk_46__container_name":"unify-query","_value_":7142,"_timestamp_":1744669860000},{"__ext__bk_46__container_name":"unify-query","_value_":9153,"_timestamp_":1744669890000},{"__ext__bk_46__container_name":"unify-query","_value_":4566,"_timestamp_":1744669920000},{"__ext__bk_46__container_name":"unify-query","_value_":3225,"_timestamp_":1744669950000},{"__ext__bk_46__container_name":"unify-query","_value_":4378,"_timestamp_":1744669980000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744670010000},{"__ext__bk_46__container_name":"unify-query","_value_":4266,"_timestamp_":1744670040000},{"__ext__bk_46__container_name":"unify-query","_value_":3645,"_timestamp_":1744670070000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744670100000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744670130000},{"__ext__bk_46__container_name":"unify-query","_value_":4333,"_timestamp_":1744670160000},{"__ext__bk_46__container_name":"unify-query","_value_":3489,"_timestamp_":1744670190000},{"__ext__bk_46__container_name":"unify-query","_value_":4303,"_timestamp_":1744670220000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744670250000},{"__ext__bk_46__container_name":"unify-query","_value_":4121,"_timestamp_":1744670280000},{"__ext__bk_46__container_name":"unify-query","_value_":3374,"_timestamp_":1744670310000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744670340000},{"__ext__bk_46__container_name":"unify-query","_value_":3242,"_timestamp_":1744670370000},{"__ext__bk_46__container_name":"unify-query","_value_":6416,"_timestamp_":1744670400000},{"__ext__bk_46__container_name":"unify-query","_value_":3697,"_timestamp_":1744670430000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744670460000},{"__ext__bk_46__container_name":"unify-query","_value_":3749,"_timestamp_":1744670490000},{"__ext__bk_46__container_name":"unify-query","_value_":4587,"_timestamp_":1744670520000},{"__ext__bk_46__container_name":"unify-query","_value_":3538,"_timestamp_":1744670550000},{"__ext__bk_46__container_name":"unify-query","_value_":4221,"_timestamp_":1744670580000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744670610000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744670640000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744670670000},{"__ext__bk_46__container_name":"unify-query","_value_":4848,"_timestamp_":1744670700000},{"__ext__bk_46__container_name":"unify-query","_value_":3551,"_timestamp_":1744670730000},{"__ext__bk_46__container_name":"unify-query","_value_":4068,"_timestamp_":1744670760000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744670790000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744670820000},{"__ext__bk_46__container_name":"unify-query","_value_":3635,"_timestamp_":1744670850000},{"__ext__bk_46__container_name":"unify-query","_value_":4256,"_timestamp_":1744670880000},{"__ext__bk_46__container_name":"unify-query","_value_":3690,"_timestamp_":1744670910000},{"__ext__bk_46__container_name":"unify-query","_value_":4155,"_timestamp_":1744670940000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744670970000},{"__ext__bk_46__container_name":"unify-query","_value_":4661,"_timestamp_":1744671000000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744671030000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744671060000},{"__ext__bk_46__container_name":"unify-query","_value_":3643,"_timestamp_":1744671090000},{"__ext__bk_46__container_name":"unify-query","_value_":4755,"_timestamp_":1744671120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744671150000},{"__ext__bk_46__container_name":"unify-query","_value_":4284,"_timestamp_":1744671180000},{"__ext__bk_46__container_name":"unify-query","_value_":3258,"_timestamp_":1744671210000},{"__ext__bk_46__container_name":"unify-query","_value_":4453,"_timestamp_":1744671240000},{"__ext__bk_46__container_name":"unify-query","_value_":3431,"_timestamp_":1744671270000},{"__ext__bk_46__container_name":"unify-query","_value_":4231,"_timestamp_":1744671300000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744671330000},{"__ext__bk_46__container_name":"unify-query","_value_":3907,"_timestamp_":1744671360000},{"__ext__bk_46__container_name":"unify-query","_value_":3524,"_timestamp_":1744671390000},{"__ext__bk_46__container_name":"unify-query","_value_":4438,"_timestamp_":1744671420000},{"__ext__bk_46__container_name":"unify-query","_value_":3547,"_timestamp_":1744671450000},{"__ext__bk_46__container_name":"unify-query","_value_":4033,"_timestamp_":1744671480000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744671510000},{"__ext__bk_46__container_name":"unify-query","_value_":4162,"_timestamp_":1744671540000},{"__ext__bk_46__container_name":"unify-query","_value_":3588,"_timestamp_":1744671570000},{"__ext__bk_46__container_name":"unify-query","_value_":16444,"_timestamp_":1744671600000},{"__ext__bk_46__container_name":"unify-query","_value_":15396,"_timestamp_":1744671630000},{"__ext__bk_46__container_name":"unify-query","_value_":3024,"_timestamp_":1744671660000},{"__ext__bk_46__container_name":"unify-query","_value_":12656,"_timestamp_":1744671690000},{"__ext__bk_46__container_name":"unify-query","_value_":4733,"_timestamp_":1744671720000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744671750000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744671780000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744671810000},{"__ext__bk_46__container_name":"unify-query","_value_":4487,"_timestamp_":1744671840000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744671870000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744671900000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744671930000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744671960000},{"__ext__bk_46__container_name":"unify-query","_value_":3472,"_timestamp_":1744671990000},{"__ext__bk_46__container_name":"unify-query","_value_":4309,"_timestamp_":1744672020000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744672050000},{"__ext__bk_46__container_name":"unify-query","_value_":4191,"_timestamp_":1744672080000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744672110000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744672140000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744672170000},{"__ext__bk_46__container_name":"unify-query","_value_":4445,"_timestamp_":1744672200000},{"__ext__bk_46__container_name":"unify-query","_value_":3822,"_timestamp_":1744672230000},{"__ext__bk_46__container_name":"unify-query","_value_":4346,"_timestamp_":1744672260000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744672290000},{"__ext__bk_46__container_name":"unify-query","_value_":4615,"_timestamp_":1744672320000},{"__ext__bk_46__container_name":"unify-query","_value_":3591,"_timestamp_":1744672350000},{"__ext__bk_46__container_name":"unify-query","_value_":4056,"_timestamp_":1744672380000},{"__ext__bk_46__container_name":"unify-query","_value_":3544,"_timestamp_":1744672410000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744672440000},{"__ext__bk_46__container_name":"unify-query","_value_":3647,"_timestamp_":1744672470000},{"__ext__bk_46__container_name":"unify-query","_value_":4887,"_timestamp_":1744672500000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744672530000},{"__ext__bk_46__container_name":"unify-query","_value_":4302,"_timestamp_":1744672560000},{"__ext__bk_46__container_name":"unify-query","_value_":3425,"_timestamp_":1744672590000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744672620000},{"__ext__bk_46__container_name":"unify-query","_value_":3532,"_timestamp_":1744672650000},{"__ext__bk_46__container_name":"unify-query","_value_":4282,"_timestamp_":1744672680000},{"__ext__bk_46__container_name":"unify-query","_value_":3571,"_timestamp_":1744672710000},{"__ext__bk_46__container_name":"unify-query","_value_":4182,"_timestamp_":1744672740000},{"__ext__bk_46__container_name":"unify-query","_value_":3210,"_timestamp_":1744672770000},{"__ext__bk_46__container_name":"unify-query","_value_":6383,"_timestamp_":1744672800000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744672830000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744672860000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744672890000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744672920000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744672950000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744672980000},{"__ext__bk_46__container_name":"unify-query","_value_":3534,"_timestamp_":1744673010000},{"__ext__bk_46__container_name":"unify-query","_value_":4212,"_timestamp_":1744673040000},{"__ext__bk_46__container_name":"unify-query","_value_":3380,"_timestamp_":1744673070000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744673100000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744673130000},{"__ext__bk_46__container_name":"unify-query","_value_":4120,"_timestamp_":1744673160000},{"__ext__bk_46__container_name":"unify-query","_value_":3526,"_timestamp_":1744673190000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744673220000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744673250000},{"__ext__bk_46__container_name":"unify-query","_value_":4370,"_timestamp_":1744673280000},{"__ext__bk_46__container_name":"unify-query","_value_":3462,"_timestamp_":1744673310000},{"__ext__bk_46__container_name":"unify-query","_value_":4223,"_timestamp_":1744673340000},{"__ext__bk_46__container_name":"unify-query","_value_":3564,"_timestamp_":1744673370000},{"__ext__bk_46__container_name":"unify-query","_value_":12072,"_timestamp_":1744673400000},{"__ext__bk_46__container_name":"unify-query","_value_":17986,"_timestamp_":1744673430000},{"__ext__bk_46__container_name":"unify-query","_value_":4089,"_timestamp_":1744673460000},{"__ext__bk_46__container_name":"unify-query","_value_":12000,"_timestamp_":1744673490000},{"__ext__bk_46__container_name":"unify-query","_value_":4790,"_timestamp_":1744673520000},{"__ext__bk_46__container_name":"unify-query","_value_":3637,"_timestamp_":1744673550000},{"__ext__bk_46__container_name":"unify-query","_value_":4177,"_timestamp_":1744673580000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744673610000},{"__ext__bk_46__container_name":"unify-query","_value_":4465,"_timestamp_":1744673640000},{"__ext__bk_46__container_name":"unify-query","_value_":3627,"_timestamp_":1744673670000},{"__ext__bk_46__container_name":"unify-query","_value_":4131,"_timestamp_":1744673700000},{"__ext__bk_46__container_name":"unify-query","_value_":3396,"_timestamp_":1744673730000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744673760000},{"__ext__bk_46__container_name":"unify-query","_value_":3638,"_timestamp_":1744673790000},{"__ext__bk_46__container_name":"unify-query","_value_":4093,"_timestamp_":1744673820000},{"__ext__bk_46__container_name":"unify-query","_value_":3584,"_timestamp_":1744673850000},{"__ext__bk_46__container_name":"unify-query","_value_":4082,"_timestamp_":1744673880000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744673910000},{"__ext__bk_46__container_name":"unify-query","_value_":4051,"_timestamp_":1744673940000},{"__ext__bk_46__container_name":"unify-query","_value_":3354,"_timestamp_":1744673970000},{"__ext__bk_46__container_name":"unify-query","_value_":6296,"_timestamp_":1744674000000},{"__ext__bk_46__container_name":"unify-query","_value_":3473,"_timestamp_":1744674030000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744674060000},{"__ext__bk_46__container_name":"unify-query","_value_":3793,"_timestamp_":1744674090000},{"__ext__bk_46__container_name":"unify-query","_value_":4391,"_timestamp_":1744674120000},{"__ext__bk_46__container_name":"unify-query","_value_":3836,"_timestamp_":1744674150000},{"__ext__bk_46__container_name":"unify-query","_value_":4190,"_timestamp_":1744674180000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674210000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744674240000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744674270000},{"__ext__bk_46__container_name":"unify-query","_value_":4964,"_timestamp_":1744674300000},{"__ext__bk_46__container_name":"unify-query","_value_":3455,"_timestamp_":1744674330000},{"__ext__bk_46__container_name":"unify-query","_value_":4116,"_timestamp_":1744674360000},{"__ext__bk_46__container_name":"unify-query","_value_":3250,"_timestamp_":1744674390000},{"__ext__bk_46__container_name":"unify-query","_value_":4494,"_timestamp_":1744674420000},{"__ext__bk_46__container_name":"unify-query","_value_":3326,"_timestamp_":1744674450000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744674480000},{"__ext__bk_46__container_name":"unify-query","_value_":3580,"_timestamp_":1744674510000},{"__ext__bk_46__container_name":"unify-query","_value_":4368,"_timestamp_":1744674540000},{"__ext__bk_46__container_name":"unify-query","_value_":3685,"_timestamp_":1744674570000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744674600000},{"__ext__bk_46__container_name":"unify-query","_value_":3699,"_timestamp_":1744674630000},{"__ext__bk_46__container_name":"unify-query","_value_":4513,"_timestamp_":1744674660000},{"__ext__bk_46__container_name":"unify-query","_value_":3729,"_timestamp_":1744674690000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744674720000},{"__ext__bk_46__container_name":"unify-query","_value_":3639,"_timestamp_":1744674750000},{"__ext__bk_46__container_name":"unify-query","_value_":4018,"_timestamp_":1744674780000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744674810000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744674840000},{"__ext__bk_46__container_name":"unify-query","_value_":3389,"_timestamp_":1744674870000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744674900000},{"__ext__bk_46__container_name":"unify-query","_value_":3540,"_timestamp_":1744674930000},{"__ext__bk_46__container_name":"unify-query","_value_":4106,"_timestamp_":1744674960000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674990000},{"__ext__bk_46__container_name":"unify-query","_value_":4268,"_timestamp_":1744675020000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744675050000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744675080000},{"__ext__bk_46__container_name":"unify-query","_value_":3511,"_timestamp_":1744675110000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744675140000},{"__ext__bk_46__container_name":"unify-query","_value_":3573,"_timestamp_":1744675170000},{"__ext__bk_46__container_name":"unify-query","_value_":17095,"_timestamp_":1744675200000},{"__ext__bk_46__container_name":"unify-query","_value_":14907,"_timestamp_":1744675230000},{"__ext__bk_46__container_name":"unify-query","_value_":6455,"_timestamp_":1744675260000},{"__ext__bk_46__container_name":"unify-query","_value_":9818,"_timestamp_":1744675290000},{"__ext__bk_46__container_name":"unify-query","_value_":5253,"_timestamp_":1744675320000},{"__ext__bk_46__container_name":"unify-query","_value_":3567,"_timestamp_":1744675350000},{"__ext__bk_46__container_name":"unify-query","_value_":4047,"_timestamp_":1744675380000},{"__ext__bk_46__container_name":"unify-query","_value_":3342,"_timestamp_":1744675410000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744675440000},{"__ext__bk_46__container_name":"unify-query","_value_":3394,"_timestamp_":1744675470000},{"__ext__bk_46__container_name":"unify-query","_value_":4260,"_timestamp_":1744675500000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744675530000},{"__ext__bk_46__container_name":"unify-query","_value_":4341,"_timestamp_":1744675560000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744675590000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744675620000},{"__ext__bk_46__container_name":"unify-query","_value_":3519,"_timestamp_":1744675650000},{"__ext__bk_46__container_name":"unify-query","_value_":4143,"_timestamp_":1744675680000},{"__ext__bk_46__container_name":"unify-query","_value_":3630,"_timestamp_":1744675710000},{"__ext__bk_46__container_name":"unify-query","_value_":4042,"_timestamp_":1744675740000},{"__ext__bk_46__container_name":"unify-query","_value_":3653,"_timestamp_":1744675770000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744675800000},{"__ext__bk_46__container_name":"unify-query","_value_":3688,"_timestamp_":1744675830000},{"__ext__bk_46__container_name":"unify-query","_value_":4450,"_timestamp_":1744675860000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744675890000},{"__ext__bk_46__container_name":"unify-query","_value_":4864,"_timestamp_":1744675920000},{"__ext__bk_46__container_name":"unify-query","_value_":3629,"_timestamp_":1744675950000},{"__ext__bk_46__container_name":"unify-query","_value_":4127,"_timestamp_":1744675980000},{"__ext__bk_46__container_name":"unify-query","_value_":3424,"_timestamp_":1744676010000},{"__ext__bk_46__container_name":"unify-query","_value_":4267,"_timestamp_":1744676040000},{"__ext__bk_46__container_name":"unify-query","_value_":3328,"_timestamp_":1744676070000},{"__ext__bk_46__container_name":"unify-query","_value_":5128,"_timestamp_":1744676100000},{"__ext__bk_46__container_name":"unify-query","_value_":3657,"_timestamp_":1744676130000},{"__ext__bk_46__container_name":"unify-query","_value_":4185,"_timestamp_":1744676160000},{"__ext__bk_46__container_name":"unify-query","_value_":3336,"_timestamp_":1744676190000},{"__ext__bk_46__container_name":"unify-query","_value_":4532,"_timestamp_":1744676220000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744676250000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744676280000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744676310000},{"__ext__bk_46__container_name":"unify-query","_value_":4463,"_timestamp_":1744676340000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676370000},{"__ext__bk_46__container_name":"unify-query","_value_":6064,"_timestamp_":1744676400000},{"__ext__bk_46__container_name":"unify-query","_value_":3292,"_timestamp_":1744676430000},{"__ext__bk_46__container_name":"unify-query","_value_":4858,"_timestamp_":1744676460000},{"__ext__bk_46__container_name":"unify-query","_value_":3543,"_timestamp_":1744676490000},{"__ext__bk_46__container_name":"unify-query","_value_":4620,"_timestamp_":1744676520000},{"__ext__bk_46__container_name":"unify-query","_value_":3750,"_timestamp_":1744676550000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744676580000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744676610000},{"__ext__bk_46__container_name":"unify-query","_value_":4152,"_timestamp_":1744676640000},{"__ext__bk_46__container_name":"unify-query","_value_":3550,"_timestamp_":1744676670000},{"__ext__bk_46__container_name":"unify-query","_value_":4011,"_timestamp_":1744676700000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676730000},{"__ext__bk_46__container_name":"unify-query","_value_":4050,"_timestamp_":1744676760000},{"__ext__bk_46__container_name":"unify-query","_value_":3118,"_timestamp_":1744676790000},{"__ext__bk_46__container_name":"unify-query","_value_":4628,"_timestamp_":1744676820000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744676850000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744676880000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744676910000},{"__ext__bk_46__container_name":"unify-query","_value_":4160,"_timestamp_":1744676940000},{"__ext__bk_46__container_name":"unify-query","_value_":3662,"_timestamp_":1744676970000},{"__ext__bk_46__container_name":"unify-query","_value_":11392,"_timestamp_":1744677000000},{"__ext__bk_46__container_name":"unify-query","_value_":18649,"_timestamp_":1744677030000},{"__ext__bk_46__container_name":"unify-query","_value_":7107,"_timestamp_":1744677060000},{"__ext__bk_46__container_name":"unify-query","_value_":9213,"_timestamp_":1744677090000},{"__ext__bk_46__container_name":"unify-query","_value_":4235,"_timestamp_":1744677120000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744677150000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744677180000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744677210000},{"__ext__bk_46__container_name":"unify-query","_value_":4233,"_timestamp_":1744677240000},{"__ext__bk_46__container_name":"unify-query","_value_":3440,"_timestamp_":1744677270000},{"__ext__bk_46__container_name":"unify-query","_value_":4383,"_timestamp_":1744677300000},{"__ext__bk_46__container_name":"unify-query","_value_":3507,"_timestamp_":1744677330000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744677360000},{"__ext__bk_46__container_name":"unify-query","_value_":3197,"_timestamp_":1744677390000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744677420000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744677450000},{"__ext__bk_46__container_name":"unify-query","_value_":4421,"_timestamp_":1744677480000},{"__ext__bk_46__container_name":"unify-query","_value_":2998,"_timestamp_":1744677510000},{"__ext__bk_46__container_name":"unify-query","_value_":4700,"_timestamp_":1744677540000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744677570000},{"__ext__bk_46__container_name":"unify-query","_value_":5781,"_timestamp_":1744677600000},{"__ext__bk_46__container_name":"unify-query","_value_":3734,"_timestamp_":1744677630000},{"__ext__bk_46__container_name":"unify-query","_value_":4510,"_timestamp_":1744677660000},{"__ext__bk_46__container_name":"unify-query","_value_":3752,"_timestamp_":1744677690000},{"__ext__bk_46__container_name":"unify-query","_value_":4447,"_timestamp_":1744677720000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744677750000},{"__ext__bk_46__container_name":"unify-query","_value_":4187,"_timestamp_":1744677780000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744677810000},{"__ext__bk_46__container_name":"unify-query","_value_":3900,"_timestamp_":1744677840000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744677870000},{"__ext__bk_46__container_name":"unify-query","_value_":4863,"_timestamp_":1744677900000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744677930000},{"__ext__bk_46__container_name":"unify-query","_value_":4335,"_timestamp_":1744677960000},{"__ext__bk_46__container_name":"unify-query","_value_":3533,"_timestamp_":1744677990000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744678020000},{"__ext__bk_46__container_name":"unify-query","_value_":3556,"_timestamp_":1744678050000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744678080000},{"__ext__bk_46__container_name":"unify-query","_value_":3664,"_timestamp_":1744678110000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744678140000},{"__ext__bk_46__container_name":"unify-query","_value_":3222,"_timestamp_":1744678170000},{"__ext__bk_46__container_name":"unify-query","_value_":4750,"_timestamp_":1744678200000},{"__ext__bk_46__container_name":"unify-query","_value_":3546,"_timestamp_":1744678230000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744678260000},{"__ext__bk_46__container_name":"unify-query","_value_":3702,"_timestamp_":1744678290000},{"__ext__bk_46__container_name":"unify-query","_value_":4564,"_timestamp_":1744678320000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744678350000},{"__ext__bk_46__container_name":"unify-query","_value_":4130,"_timestamp_":1744678380000},{"__ext__bk_46__container_name":"unify-query","_value_":3412,"_timestamp_":1744678410000},{"__ext__bk_46__container_name":"unify-query","_value_":4614,"_timestamp_":1744678440000},{"__ext__bk_46__container_name":"unify-query","_value_":3522,"_timestamp_":1744678470000},{"__ext__bk_46__container_name":"unify-query","_value_":4148,"_timestamp_":1744678500000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744678530000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744678560000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744678590000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744678620000},{"__ext__bk_46__container_name":"unify-query","_value_":3529,"_timestamp_":1744678650000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744678680000},{"__ext__bk_46__container_name":"unify-query","_value_":3487,"_timestamp_":1744678710000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744678740000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744678770000},{"__ext__bk_46__container_name":"unify-query","_value_":7230,"_timestamp_":1744678800000},{"__ext__bk_46__container_name":"unify-query","_value_":3818,"_timestamp_":1744678830000},{"__ext__bk_46__container_name":"unify-query","_value_":11924,"_timestamp_":1744678860000},{"__ext__bk_46__container_name":"unify-query","_value_":27269,"_timestamp_":1744678890000},{"__ext__bk_46__container_name":"unify-query","_value_":5073,"_timestamp_":1744678920000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744678950000},{"__ext__bk_46__container_name":"unify-query","_value_":4474,"_timestamp_":1744678980000},{"__ext__bk_46__container_name":"unify-query","_value_":3536,"_timestamp_":1744679010000},{"__ext__bk_46__container_name":"unify-query","_value_":4525,"_timestamp_":1744679040000},{"__ext__bk_46__container_name":"unify-query","_value_":3503,"_timestamp_":1744679070000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744679100000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744679130000},{"__ext__bk_46__container_name":"unify-query","_value_":4259,"_timestamp_":1744679160000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744679190000},{"__ext__bk_46__container_name":"unify-query","_value_":4218,"_timestamp_":1744679220000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744679250000},{"__ext__bk_46__container_name":"unify-query","_value_":4417,"_timestamp_":1744679280000},{"__ext__bk_46__container_name":"unify-query","_value_":3730,"_timestamp_":1744679310000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744679340000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744679370000},{"__ext__bk_46__container_name":"unify-query","_value_":4849,"_timestamp_":1744679400000},{"__ext__bk_46__container_name":"unify-query","_value_":3803,"_timestamp_":1744679430000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744679460000},{"__ext__bk_46__container_name":"unify-query","_value_":3674,"_timestamp_":1744679490000},{"__ext__bk_46__container_name":"unify-query","_value_":4727,"_timestamp_":1744679520000},{"__ext__bk_46__container_name":"unify-query","_value_":3926,"_timestamp_":1744679550000},{"__ext__bk_46__container_name":"unify-query","_value_":4173,"_timestamp_":1744679580000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744679610000},{"__ext__bk_46__container_name":"unify-query","_value_":4968,"_timestamp_":1744679640000},{"__ext__bk_46__container_name":"unify-query","_value_":3432,"_timestamp_":1744679670000},{"__ext__bk_46__container_name":"unify-query","_value_":5059,"_timestamp_":1744679700000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744679730000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744679760000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744679790000},{"__ext__bk_46__container_name":"unify-query","_value_":4436,"_timestamp_":1744679820000},{"__ext__bk_46__container_name":"unify-query","_value_":5299,"_timestamp_":1744679850000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744679880000},{"__ext__bk_46__container_name":"unify-query","_value_":3861,"_timestamp_":1744679910000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744679940000},{"__ext__bk_46__container_name":"unify-query","_value_":3711,"_timestamp_":1744679970000},{"__ext__bk_46__container_name":"unify-query","_value_":6021,"_timestamp_":1744680000000},{"__ext__bk_46__container_name":"unify-query","_value_":3942,"_timestamp_":1744680030000},{"__ext__bk_46__container_name":"unify-query","_value_":4800,"_timestamp_":1744680060000},{"__ext__bk_46__container_name":"unify-query","_value_":3681,"_timestamp_":1744680090000},{"__ext__bk_46__container_name":"unify-query","_value_":4592,"_timestamp_":1744680120000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744680150000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744680180000},{"__ext__bk_46__container_name":"unify-query","_value_":3490,"_timestamp_":1744680210000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744680240000},{"__ext__bk_46__container_name":"unify-query","_value_":4009,"_timestamp_":1744680270000},{"__ext__bk_46__container_name":"unify-query","_value_":4837,"_timestamp_":1744680300000},{"__ext__bk_46__container_name":"unify-query","_value_":3227,"_timestamp_":1744680330000},{"__ext__bk_46__container_name":"unify-query","_value_":4531,"_timestamp_":1744680360000},{"__ext__bk_46__container_name":"unify-query","_value_":2888,"_timestamp_":1744680390000},{"__ext__bk_46__container_name":"unify-query","_value_":5083,"_timestamp_":1744680420000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744680450000},{"__ext__bk_46__container_name":"unify-query","_value_":4207,"_timestamp_":1744680480000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744680510000},{"__ext__bk_46__container_name":"unify-query","_value_":4482,"_timestamp_":1744680540000},{"__ext__bk_46__container_name":"unify-query","_value_":3110,"_timestamp_":1744680570000},{"__ext__bk_46__container_name":"unify-query","_value_":13551,"_timestamp_":1744680600000},{"__ext__bk_46__container_name":"unify-query","_value_":17159,"_timestamp_":1744680630000},{"__ext__bk_46__container_name":"unify-query","_value_":6284,"_timestamp_":1744680660000},{"__ext__bk_46__container_name":"unify-query","_value_":9924,"_timestamp_":1744680690000},{"__ext__bk_46__container_name":"unify-query","_value_":4547,"_timestamp_":1744680720000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744680750000},{"__ext__bk_46__container_name":"unify-query","_value_":4312,"_timestamp_":1744680780000},{"__ext__bk_46__container_name":"unify-query","_value_":3689,"_timestamp_":1744680810000},{"__ext__bk_46__container_name":"unify-query","_value_":4680,"_timestamp_":1744680840000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744680870000},{"__ext__bk_46__container_name":"unify-query","_value_":4886,"_timestamp_":1744680900000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744680930000},{"__ext__bk_46__container_name":"unify-query","_value_":4810,"_timestamp_":1744680960000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744680990000},{"__ext__bk_46__container_name":"unify-query","_value_":4594,"_timestamp_":1744681020000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744681050000},{"__ext__bk_46__container_name":"unify-query","_value_":4562,"_timestamp_":1744681080000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744681110000},{"__ext__bk_46__container_name":"unify-query","_value_":5243,"_timestamp_":1744681140000},{"__ext__bk_46__container_name":"unify-query","_value_":5135,"_timestamp_":1744681170000},{"__ext__bk_46__container_name":"unify-query","_value_":6671,"_timestamp_":1744681200000},{"__ext__bk_46__container_name":"unify-query","_value_":3806,"_timestamp_":1744681230000},{"__ext__bk_46__container_name":"unify-query","_value_":4535,"_timestamp_":1744681260000},{"__ext__bk_46__container_name":"unify-query","_value_":3721,"_timestamp_":1744681290000},{"__ext__bk_46__container_name":"unify-query","_value_":4799,"_timestamp_":1744681320000},{"__ext__bk_46__container_name":"unify-query","_value_":3909,"_timestamp_":1744681350000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744681380000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744681410000},{"__ext__bk_46__container_name":"unify-query","_value_":4359,"_timestamp_":1744681440000},{"__ext__bk_46__container_name":"unify-query","_value_":4063,"_timestamp_":1744681470000},{"__ext__bk_46__container_name":"unify-query","_value_":5231,"_timestamp_":1744681500000},{"__ext__bk_46__container_name":"unify-query","_value_":3778,"_timestamp_":1744681530000},{"__ext__bk_46__container_name":"unify-query","_value_":4684,"_timestamp_":1744681560000},{"__ext__bk_46__container_name":"unify-query","_value_":4072,"_timestamp_":1744681590000},{"__ext__bk_46__container_name":"unify-query","_value_":5029,"_timestamp_":1744681620000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744681650000},{"__ext__bk_46__container_name":"unify-query","_value_":4670,"_timestamp_":1744681680000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744681710000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744681740000},{"__ext__bk_46__container_name":"unify-query","_value_":3041,"_timestamp_":1744681770000},{"__ext__bk_46__container_name":"unify-query","_value_":5043,"_timestamp_":1744681800000},{"__ext__bk_46__container_name":"unify-query","_value_":3530,"_timestamp_":1744681830000},{"__ext__bk_46__container_name":"unify-query","_value_":6807,"_timestamp_":1744681860000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744681890000},{"__ext__bk_46__container_name":"unify-query","_value_":6841,"_timestamp_":1744681920000},{"__ext__bk_46__container_name":"unify-query","_value_":4519,"_timestamp_":1744681950000},{"__ext__bk_46__container_name":"unify-query","_value_":6617,"_timestamp_":1744681980000},{"__ext__bk_46__container_name":"unify-query","_value_":4633,"_timestamp_":1744682010000},{"__ext__bk_46__container_name":"unify-query","_value_":5997,"_timestamp_":1744682040000},{"__ext__bk_46__container_name":"unify-query","_value_":4446,"_timestamp_":1744682070000},{"__ext__bk_46__container_name":"unify-query","_value_":5569,"_timestamp_":1744682100000},{"__ext__bk_46__container_name":"unify-query","_value_":4324,"_timestamp_":1744682130000},{"__ext__bk_46__container_name":"unify-query","_value_":5354,"_timestamp_":1744682160000},{"__ext__bk_46__container_name":"unify-query","_value_":7245,"_timestamp_":1744682190000},{"__ext__bk_46__container_name":"unify-query","_value_":5258,"_timestamp_":1744682220000},{"__ext__bk_46__container_name":"unify-query","_value_":4296,"_timestamp_":1744682250000},{"__ext__bk_46__container_name":"unify-query","_value_":5349,"_timestamp_":1744682280000},{"__ext__bk_46__container_name":"unify-query","_value_":4479,"_timestamp_":1744682310000},{"__ext__bk_46__container_name":"unify-query","_value_":5127,"_timestamp_":1744682340000},{"__ext__bk_46__container_name":"unify-query","_value_":4006,"_timestamp_":1744682370000},{"__ext__bk_46__container_name":"unify-query","_value_":19058,"_timestamp_":1744682400000},{"__ext__bk_46__container_name":"unify-query","_value_":14501,"_timestamp_":1744682430000},{"__ext__bk_46__container_name":"unify-query","_value_":3810,"_timestamp_":1744682460000},{"__ext__bk_46__container_name":"unify-query","_value_":12368,"_timestamp_":1744682490000},{"__ext__bk_46__container_name":"unify-query","_value_":6976,"_timestamp_":1744682520000},{"__ext__bk_46__container_name":"unify-query","_value_":4399,"_timestamp_":1744682550000},{"__ext__bk_46__container_name":"unify-query","_value_":5482,"_timestamp_":1744682580000},{"__ext__bk_46__container_name":"unify-query","_value_":4524,"_timestamp_":1744682610000},{"__ext__bk_46__container_name":"unify-query","_value_":5478,"_timestamp_":1744682640000},{"__ext__bk_46__container_name":"unify-query","_value_":4920,"_timestamp_":1744682670000},{"__ext__bk_46__container_name":"unify-query","_value_":5347,"_timestamp_":1744682700000},{"__ext__bk_46__container_name":"unify-query","_value_":4427,"_timestamp_":1744682730000},{"__ext__bk_46__container_name":"unify-query","_value_":5102,"_timestamp_":1744682760000},{"__ext__bk_46__container_name":"unify-query","_value_":4441,"_timestamp_":1744682790000},{"__ext__bk_46__container_name":"unify-query","_value_":5596,"_timestamp_":1744682820000},{"__ext__bk_46__container_name":"unify-query","_value_":4888,"_timestamp_":1744682850000},{"__ext__bk_46__container_name":"unify-query","_value_":5306,"_timestamp_":1744682880000},{"__ext__bk_46__container_name":"unify-query","_value_":4825,"_timestamp_":1744682910000},{"__ext__bk_46__container_name":"unify-query","_value_":5897,"_timestamp_":1744682940000},{"__ext__bk_46__container_name":"unify-query","_value_":4481,"_timestamp_":1744682970000},{"__ext__bk_46__container_name":"unify-query","_value_":6086,"_timestamp_":1744683000000},{"__ext__bk_46__container_name":"unify-query","_value_":4910,"_timestamp_":1744683030000},{"__ext__bk_46__container_name":"unify-query","_value_":5676,"_timestamp_":1744683060000},{"__ext__bk_46__container_name":"unify-query","_value_":3626,"_timestamp_":1744683090000},{"__ext__bk_46__container_name":"unify-query","_value_":6929,"_timestamp_":1744683120000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744683150000},{"__ext__bk_46__container_name":"unify-query","_value_":5525,"_timestamp_":1744683180000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744683210000},{"__ext__bk_46__container_name":"unify-query","_value_":5617,"_timestamp_":1744683240000},{"__ext__bk_46__container_name":"unify-query","_value_":4503,"_timestamp_":1744683270000},{"__ext__bk_46__container_name":"unify-query","_value_":6328,"_timestamp_":1744683300000},{"__ext__bk_46__container_name":"unify-query","_value_":4557,"_timestamp_":1744683330000},{"__ext__bk_46__container_name":"unify-query","_value_":5356,"_timestamp_":1744683360000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744683390000},{"__ext__bk_46__container_name":"unify-query","_value_":5335,"_timestamp_":1744683420000},{"__ext__bk_46__container_name":"unify-query","_value_":4640,"_timestamp_":1744683450000},{"__ext__bk_46__container_name":"unify-query","_value_":5399,"_timestamp_":1744683480000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744683510000},{"__ext__bk_46__container_name":"unify-query","_value_":5415,"_timestamp_":1744683540000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744683570000},{"__ext__bk_46__container_name":"unify-query","_value_":6949,"_timestamp_":1744683600000},{"__ext__bk_46__container_name":"unify-query","_value_":4574,"_timestamp_":1744683630000},{"__ext__bk_46__container_name":"unify-query","_value_":5757,"_timestamp_":1744683660000},{"__ext__bk_46__container_name":"unify-query","_value_":4669,"_timestamp_":1744683690000},{"__ext__bk_46__container_name":"unify-query","_value_":5706,"_timestamp_":1744683720000},{"__ext__bk_46__container_name":"unify-query","_value_":4472,"_timestamp_":1744683750000},{"__ext__bk_46__container_name":"unify-query","_value_":5386,"_timestamp_":1744683780000},{"__ext__bk_46__container_name":"unify-query","_value_":4490,"_timestamp_":1744683810000},{"__ext__bk_46__container_name":"unify-query","_value_":5104,"_timestamp_":1744683840000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744683870000},{"__ext__bk_46__container_name":"unify-query","_value_":5979,"_timestamp_":1744683900000},{"__ext__bk_46__container_name":"unify-query","_value_":4853,"_timestamp_":1744683930000},{"__ext__bk_46__container_name":"unify-query","_value_":6691,"_timestamp_":1744683960000},{"__ext__bk_46__container_name":"unify-query","_value_":4572,"_timestamp_":1744683990000},{"__ext__bk_46__container_name":"unify-query","_value_":5554,"_timestamp_":1744684020000},{"__ext__bk_46__container_name":"unify-query","_value_":5244,"_timestamp_":1744684050000},{"__ext__bk_46__container_name":"unify-query","_value_":5392,"_timestamp_":1744684080000},{"__ext__bk_46__container_name":"unify-query","_value_":4550,"_timestamp_":1744684110000},{"__ext__bk_46__container_name":"unify-query","_value_":520,"_timestamp_":1744684140000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":52,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":8,"connect_db":55,"match_query_routing_rule":0,"check_permission":73,"check_query_semantic":0,"pick_valid_storage":1},"total_record_size":269248,"timetaken":0.191,"result_schema":[{"field_type":"string","field_name":"__c0","field_alias":"__ext__bk_46__container_name","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1},{"field_type":"long","field_name":"__c2","field_alias":"_timestamp_","field_index":2}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SELECT CAST(__ext[\"container_name\"] AS STRING) AS `__ext__bk_46__container_name`, COUNT(`gseIndex`) AS `_value_`, CAST(dtEventTimeStamp / 30000 AS INT) * 30000 AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1744662509999 AND `dtEventTimeStamp` <= 1744684142999 AND `thedate` = '20250415' GROUP BY __ext__bk_46__container_name, _timestamp_ ORDER BY `_timestamp_` ASC, `_value_` ASC LIMIT 2000005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041500","end":"2025041523"}},"cluster":"doris-test","totalRecords":722,"external_api_call_time_mills":{"bkbase_auth_api":72,"bkbase_meta_api":6,"bkbase_apigw_api":28},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"__ext__bk_46__container_name":"unify-query","_value_":3684,"_timestamp_":1744662510000},{"__ext__bk_46__container_name":"unify-query","_value_":4012,"_timestamp_":1744662540000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744662570000},{"__ext__bk_46__container_name":"unify-query","_value_":17092,"_timestamp_":1744662600000},{"__ext__bk_46__container_name":"unify-query","_value_":12881,"_timestamp_":1744662630000},{"__ext__bk_46__container_name":"unify-query","_value_":5902,"_timestamp_":1744662660000},{"__ext__bk_46__container_name":"unify-query","_value_":10443,"_timestamp_":1744662690000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744662720000},{"__ext__bk_46__container_name":"unify-query","_value_":3357,"_timestamp_":1744662750000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744662780000},{"__ext__bk_46__container_name":"unify-query","_value_":3683,"_timestamp_":1744662810000},{"__ext__bk_46__container_name":"unify-query","_value_":4353,"_timestamp_":1744662840000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744662870000},{"__ext__bk_46__container_name":"unify-query","_value_":4251,"_timestamp_":1744662900000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744662930000},{"__ext__bk_46__container_name":"unify-query","_value_":4036,"_timestamp_":1744662960000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744662990000},{"__ext__bk_46__container_name":"unify-query","_value_":4351,"_timestamp_":1744663020000},{"__ext__bk_46__container_name":"unify-query","_value_":3651,"_timestamp_":1744663050000},{"__ext__bk_46__container_name":"unify-query","_value_":4096,"_timestamp_":1744663080000},{"__ext__bk_46__container_name":"unify-query","_value_":3618,"_timestamp_":1744663110000},{"__ext__bk_46__container_name":"unify-query","_value_":4100,"_timestamp_":1744663140000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744663170000},{"__ext__bk_46__container_name":"unify-query","_value_":6044,"_timestamp_":1744663200000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744663230000},{"__ext__bk_46__container_name":"unify-query","_value_":4461,"_timestamp_":1744663260000},{"__ext__bk_46__container_name":"unify-query","_value_":3783,"_timestamp_":1744663290000},{"__ext__bk_46__container_name":"unify-query","_value_":4559,"_timestamp_":1744663320000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744663350000},{"__ext__bk_46__container_name":"unify-query","_value_":3869,"_timestamp_":1744663380000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744663410000},{"__ext__bk_46__container_name":"unify-query","_value_":4473,"_timestamp_":1744663440000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744663470000},{"__ext__bk_46__container_name":"unify-query","_value_":4923,"_timestamp_":1744663500000},{"__ext__bk_46__container_name":"unify-query","_value_":3379,"_timestamp_":1744663530000},{"__ext__bk_46__container_name":"unify-query","_value_":4489,"_timestamp_":1744663560000},{"__ext__bk_46__container_name":"unify-query","_value_":3411,"_timestamp_":1744663590000},{"__ext__bk_46__container_name":"unify-query","_value_":4374,"_timestamp_":1744663620000},{"__ext__bk_46__container_name":"unify-query","_value_":3370,"_timestamp_":1744663650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744663680000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744663710000},{"__ext__bk_46__container_name":"unify-query","_value_":4318,"_timestamp_":1744663740000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744663770000},{"__ext__bk_46__container_name":"unify-query","_value_":4334,"_timestamp_":1744663800000},{"__ext__bk_46__container_name":"unify-query","_value_":3767,"_timestamp_":1744663830000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744663860000},{"__ext__bk_46__container_name":"unify-query","_value_":3703,"_timestamp_":1744663890000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744663920000},{"__ext__bk_46__container_name":"unify-query","_value_":3667,"_timestamp_":1744663950000},{"__ext__bk_46__container_name":"unify-query","_value_":3998,"_timestamp_":1744663980000},{"__ext__bk_46__container_name":"unify-query","_value_":3579,"_timestamp_":1744664010000},{"__ext__bk_46__container_name":"unify-query","_value_":4156,"_timestamp_":1744664040000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744664070000},{"__ext__bk_46__container_name":"unify-query","_value_":4344,"_timestamp_":1744664100000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744664130000},{"__ext__bk_46__container_name":"unify-query","_value_":4161,"_timestamp_":1744664160000},{"__ext__bk_46__container_name":"unify-query","_value_":3484,"_timestamp_":1744664190000},{"__ext__bk_46__container_name":"unify-query","_value_":4273,"_timestamp_":1744664220000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744664250000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744664280000},{"__ext__bk_46__container_name":"unify-query","_value_":3619,"_timestamp_":1744664310000},{"__ext__bk_46__container_name":"unify-query","_value_":4013,"_timestamp_":1744664340000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744664370000},{"__ext__bk_46__container_name":"unify-query","_value_":18144,"_timestamp_":1744664400000},{"__ext__bk_46__container_name":"unify-query","_value_":13615,"_timestamp_":1744664430000},{"__ext__bk_46__container_name":"unify-query","_value_":3178,"_timestamp_":1744664460000},{"__ext__bk_46__container_name":"unify-query","_value_":13044,"_timestamp_":1744664490000},{"__ext__bk_46__container_name":"unify-query","_value_":4767,"_timestamp_":1744664520000},{"__ext__bk_46__container_name":"unify-query","_value_":3528,"_timestamp_":1744664550000},{"__ext__bk_46__container_name":"unify-query","_value_":4316,"_timestamp_":1744664580000},{"__ext__bk_46__container_name":"unify-query","_value_":3317,"_timestamp_":1744664610000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744664640000},{"__ext__bk_46__container_name":"unify-query","_value_":3599,"_timestamp_":1744664670000},{"__ext__bk_46__container_name":"unify-query","_value_":4149,"_timestamp_":1744664700000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744664730000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744664760000},{"__ext__bk_46__container_name":"unify-query","_value_":3384,"_timestamp_":1744664790000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744664820000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744664850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744664880000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744664910000},{"__ext__bk_46__container_name":"unify-query","_value_":4244,"_timestamp_":1744664940000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744664970000},{"__ext__bk_46__container_name":"unify-query","_value_":4305,"_timestamp_":1744665000000},{"__ext__bk_46__container_name":"unify-query","_value_":3771,"_timestamp_":1744665030000},{"__ext__bk_46__container_name":"unify-query","_value_":4485,"_timestamp_":1744665060000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744665090000},{"__ext__bk_46__container_name":"unify-query","_value_":4423,"_timestamp_":1744665120000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665150000},{"__ext__bk_46__container_name":"unify-query","_value_":4125,"_timestamp_":1744665180000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744665210000},{"__ext__bk_46__container_name":"unify-query","_value_":4252,"_timestamp_":1744665240000},{"__ext__bk_46__container_name":"unify-query","_value_":3427,"_timestamp_":1744665270000},{"__ext__bk_46__container_name":"unify-query","_value_":5089,"_timestamp_":1744665300000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744665330000},{"__ext__bk_46__container_name":"unify-query","_value_":4349,"_timestamp_":1744665360000},{"__ext__bk_46__container_name":"unify-query","_value_":3188,"_timestamp_":1744665390000},{"__ext__bk_46__container_name":"unify-query","_value_":4556,"_timestamp_":1744665420000},{"__ext__bk_46__container_name":"unify-query","_value_":3372,"_timestamp_":1744665450000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744665480000},{"__ext__bk_46__container_name":"unify-query","_value_":3445,"_timestamp_":1744665510000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744665540000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744665570000},{"__ext__bk_46__container_name":"unify-query","_value_":6235,"_timestamp_":1744665600000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744665630000},{"__ext__bk_46__container_name":"unify-query","_value_":4577,"_timestamp_":1744665660000},{"__ext__bk_46__container_name":"unify-query","_value_":3719,"_timestamp_":1744665690000},{"__ext__bk_46__container_name":"unify-query","_value_":4548,"_timestamp_":1744665720000},{"__ext__bk_46__container_name":"unify-query","_value_":3420,"_timestamp_":1744665750000},{"__ext__bk_46__container_name":"unify-query","_value_":4246,"_timestamp_":1744665780000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744665810000},{"__ext__bk_46__container_name":"unify-query","_value_":4332,"_timestamp_":1744665840000},{"__ext__bk_46__container_name":"unify-query","_value_":3422,"_timestamp_":1744665870000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744665900000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744665930000},{"__ext__bk_46__container_name":"unify-query","_value_":4119,"_timestamp_":1744665960000},{"__ext__bk_46__container_name":"unify-query","_value_":3570,"_timestamp_":1744665990000},{"__ext__bk_46__container_name":"unify-query","_value_":4144,"_timestamp_":1744666020000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744666050000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744666080000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744666110000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744666140000},{"__ext__bk_46__container_name":"unify-query","_value_":3315,"_timestamp_":1744666170000},{"__ext__bk_46__container_name":"unify-query","_value_":16721,"_timestamp_":1744666200000},{"__ext__bk_46__container_name":"unify-query","_value_":13631,"_timestamp_":1744666230000},{"__ext__bk_46__container_name":"unify-query","_value_":2982,"_timestamp_":1744666260000},{"__ext__bk_46__container_name":"unify-query","_value_":11858,"_timestamp_":1744666290000},{"__ext__bk_46__container_name":"unify-query","_value_":5515,"_timestamp_":1744666320000},{"__ext__bk_46__container_name":"unify-query","_value_":2869,"_timestamp_":1744666350000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744666380000},{"__ext__bk_46__container_name":"unify-query","_value_":3603,"_timestamp_":1744666410000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744666440000},{"__ext__bk_46__container_name":"unify-query","_value_":3264,"_timestamp_":1744666470000},{"__ext__bk_46__container_name":"unify-query","_value_":4377,"_timestamp_":1744666500000},{"__ext__bk_46__container_name":"unify-query","_value_":3443,"_timestamp_":1744666530000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744666560000},{"__ext__bk_46__container_name":"unify-query","_value_":3459,"_timestamp_":1744666590000},{"__ext__bk_46__container_name":"unify-query","_value_":4342,"_timestamp_":1744666620000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744666650000},{"__ext__bk_46__container_name":"unify-query","_value_":4052,"_timestamp_":1744666680000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744666710000},{"__ext__bk_46__container_name":"unify-query","_value_":4128,"_timestamp_":1744666740000},{"__ext__bk_46__container_name":"unify-query","_value_":3499,"_timestamp_":1744666770000},{"__ext__bk_46__container_name":"unify-query","_value_":6209,"_timestamp_":1744666800000},{"__ext__bk_46__container_name":"unify-query","_value_":3575,"_timestamp_":1744666830000},{"__ext__bk_46__container_name":"unify-query","_value_":4543,"_timestamp_":1744666860000},{"__ext__bk_46__container_name":"unify-query","_value_":3604,"_timestamp_":1744666890000},{"__ext__bk_46__container_name":"unify-query","_value_":4579,"_timestamp_":1744666920000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744666950000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744666980000},{"__ext__bk_46__container_name":"unify-query","_value_":3416,"_timestamp_":1744667010000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667040000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744667070000},{"__ext__bk_46__container_name":"unify-query","_value_":5054,"_timestamp_":1744667100000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744667130000},{"__ext__bk_46__container_name":"unify-query","_value_":4313,"_timestamp_":1744667160000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744667190000},{"__ext__bk_46__container_name":"unify-query","_value_":4118,"_timestamp_":1744667220000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744667250000},{"__ext__bk_46__container_name":"unify-query","_value_":4280,"_timestamp_":1744667280000},{"__ext__bk_46__container_name":"unify-query","_value_":3634,"_timestamp_":1744667310000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744667340000},{"__ext__bk_46__container_name":"unify-query","_value_":3807,"_timestamp_":1744667370000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744667400000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744667430000},{"__ext__bk_46__container_name":"unify-query","_value_":4630,"_timestamp_":1744667460000},{"__ext__bk_46__container_name":"unify-query","_value_":3845,"_timestamp_":1744667490000},{"__ext__bk_46__container_name":"unify-query","_value_":4361,"_timestamp_":1744667520000},{"__ext__bk_46__container_name":"unify-query","_value_":3572,"_timestamp_":1744667550000},{"__ext__bk_46__container_name":"unify-query","_value_":4095,"_timestamp_":1744667580000},{"__ext__bk_46__container_name":"unify-query","_value_":3535,"_timestamp_":1744667610000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744667640000},{"__ext__bk_46__container_name":"unify-query","_value_":3390,"_timestamp_":1744667670000},{"__ext__bk_46__container_name":"unify-query","_value_":4262,"_timestamp_":1744667700000},{"__ext__bk_46__container_name":"unify-query","_value_":3398,"_timestamp_":1744667730000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744667760000},{"__ext__bk_46__container_name":"unify-query","_value_":3429,"_timestamp_":1744667790000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744667820000},{"__ext__bk_46__container_name":"unify-query","_value_":3482,"_timestamp_":1744667850000},{"__ext__bk_46__container_name":"unify-query","_value_":4166,"_timestamp_":1744667880000},{"__ext__bk_46__container_name":"unify-query","_value_":3612,"_timestamp_":1744667910000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744667940000},{"__ext__bk_46__container_name":"unify-query","_value_":3423,"_timestamp_":1744667970000},{"__ext__bk_46__container_name":"unify-query","_value_":18203,"_timestamp_":1744668000000},{"__ext__bk_46__container_name":"unify-query","_value_":13685,"_timestamp_":1744668030000},{"__ext__bk_46__container_name":"unify-query","_value_":3281,"_timestamp_":1744668060000},{"__ext__bk_46__container_name":"unify-query","_value_":12556,"_timestamp_":1744668090000},{"__ext__bk_46__container_name":"unify-query","_value_":4893,"_timestamp_":1744668120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744668150000},{"__ext__bk_46__container_name":"unify-query","_value_":4336,"_timestamp_":1744668180000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744668210000},{"__ext__bk_46__container_name":"unify-query","_value_":4097,"_timestamp_":1744668240000},{"__ext__bk_46__container_name":"unify-query","_value_":3669,"_timestamp_":1744668270000},{"__ext__bk_46__container_name":"unify-query","_value_":3997,"_timestamp_":1744668300000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744668330000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744668360000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744668390000},{"__ext__bk_46__container_name":"unify-query","_value_":3877,"_timestamp_":1744668420000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744668450000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744668480000},{"__ext__bk_46__container_name":"unify-query","_value_":3469,"_timestamp_":1744668510000},{"__ext__bk_46__container_name":"unify-query","_value_":4243,"_timestamp_":1744668540000},{"__ext__bk_46__container_name":"unify-query","_value_":3304,"_timestamp_":1744668570000},{"__ext__bk_46__container_name":"unify-query","_value_":4690,"_timestamp_":1744668600000},{"__ext__bk_46__container_name":"unify-query","_value_":3717,"_timestamp_":1744668630000},{"__ext__bk_46__container_name":"unify-query","_value_":4618,"_timestamp_":1744668660000},{"__ext__bk_46__container_name":"unify-query","_value_":3732,"_timestamp_":1744668690000},{"__ext__bk_46__container_name":"unify-query","_value_":4477,"_timestamp_":1744668720000},{"__ext__bk_46__container_name":"unify-query","_value_":3615,"_timestamp_":1744668750000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744668780000},{"__ext__bk_46__container_name":"unify-query","_value_":3367,"_timestamp_":1744668810000},{"__ext__bk_46__container_name":"unify-query","_value_":4193,"_timestamp_":1744668840000},{"__ext__bk_46__container_name":"unify-query","_value_":3592,"_timestamp_":1744668870000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744668900000},{"__ext__bk_46__container_name":"unify-query","_value_":3359,"_timestamp_":1744668930000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744668960000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744668990000},{"__ext__bk_46__container_name":"unify-query","_value_":4375,"_timestamp_":1744669020000},{"__ext__bk_46__container_name":"unify-query","_value_":3386,"_timestamp_":1744669050000},{"__ext__bk_46__container_name":"unify-query","_value_":4281,"_timestamp_":1744669080000},{"__ext__bk_46__container_name":"unify-query","_value_":3410,"_timestamp_":1744669110000},{"__ext__bk_46__container_name":"unify-query","_value_":4545,"_timestamp_":1744669140000},{"__ext__bk_46__container_name":"unify-query","_value_":3724,"_timestamp_":1744669170000},{"__ext__bk_46__container_name":"unify-query","_value_":5903,"_timestamp_":1744669200000},{"__ext__bk_46__container_name":"unify-query","_value_":3672,"_timestamp_":1744669230000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744669260000},{"__ext__bk_46__container_name":"unify-query","_value_":3792,"_timestamp_":1744669290000},{"__ext__bk_46__container_name":"unify-query","_value_":4422,"_timestamp_":1744669320000},{"__ext__bk_46__container_name":"unify-query","_value_":3718,"_timestamp_":1744669350000},{"__ext__bk_46__container_name":"unify-query","_value_":4213,"_timestamp_":1744669380000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744669410000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744669440000},{"__ext__bk_46__container_name":"unify-query","_value_":3542,"_timestamp_":1744669470000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744669500000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669530000},{"__ext__bk_46__container_name":"unify-query","_value_":4354,"_timestamp_":1744669560000},{"__ext__bk_46__container_name":"unify-query","_value_":3368,"_timestamp_":1744669590000},{"__ext__bk_46__container_name":"unify-query","_value_":4229,"_timestamp_":1744669620000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744669650000},{"__ext__bk_46__container_name":"unify-query","_value_":4310,"_timestamp_":1744669680000},{"__ext__bk_46__container_name":"unify-query","_value_":3512,"_timestamp_":1744669710000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744669740000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744669770000},{"__ext__bk_46__container_name":"unify-query","_value_":12171,"_timestamp_":1744669800000},{"__ext__bk_46__container_name":"unify-query","_value_":18129,"_timestamp_":1744669830000},{"__ext__bk_46__container_name":"unify-query","_value_":7142,"_timestamp_":1744669860000},{"__ext__bk_46__container_name":"unify-query","_value_":9153,"_timestamp_":1744669890000},{"__ext__bk_46__container_name":"unify-query","_value_":4566,"_timestamp_":1744669920000},{"__ext__bk_46__container_name":"unify-query","_value_":3225,"_timestamp_":1744669950000},{"__ext__bk_46__container_name":"unify-query","_value_":4378,"_timestamp_":1744669980000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744670010000},{"__ext__bk_46__container_name":"unify-query","_value_":4266,"_timestamp_":1744670040000},{"__ext__bk_46__container_name":"unify-query","_value_":3645,"_timestamp_":1744670070000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744670100000},{"__ext__bk_46__container_name":"unify-query","_value_":3350,"_timestamp_":1744670130000},{"__ext__bk_46__container_name":"unify-query","_value_":4333,"_timestamp_":1744670160000},{"__ext__bk_46__container_name":"unify-query","_value_":3489,"_timestamp_":1744670190000},{"__ext__bk_46__container_name":"unify-query","_value_":4303,"_timestamp_":1744670220000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744670250000},{"__ext__bk_46__container_name":"unify-query","_value_":4121,"_timestamp_":1744670280000},{"__ext__bk_46__container_name":"unify-query","_value_":3374,"_timestamp_":1744670310000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744670340000},{"__ext__bk_46__container_name":"unify-query","_value_":3242,"_timestamp_":1744670370000},{"__ext__bk_46__container_name":"unify-query","_value_":6416,"_timestamp_":1744670400000},{"__ext__bk_46__container_name":"unify-query","_value_":3697,"_timestamp_":1744670430000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744670460000},{"__ext__bk_46__container_name":"unify-query","_value_":3749,"_timestamp_":1744670490000},{"__ext__bk_46__container_name":"unify-query","_value_":4587,"_timestamp_":1744670520000},{"__ext__bk_46__container_name":"unify-query","_value_":3538,"_timestamp_":1744670550000},{"__ext__bk_46__container_name":"unify-query","_value_":4221,"_timestamp_":1744670580000},{"__ext__bk_46__container_name":"unify-query","_value_":3476,"_timestamp_":1744670610000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744670640000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744670670000},{"__ext__bk_46__container_name":"unify-query","_value_":4848,"_timestamp_":1744670700000},{"__ext__bk_46__container_name":"unify-query","_value_":3551,"_timestamp_":1744670730000},{"__ext__bk_46__container_name":"unify-query","_value_":4068,"_timestamp_":1744670760000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744670790000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744670820000},{"__ext__bk_46__container_name":"unify-query","_value_":3635,"_timestamp_":1744670850000},{"__ext__bk_46__container_name":"unify-query","_value_":4256,"_timestamp_":1744670880000},{"__ext__bk_46__container_name":"unify-query","_value_":3690,"_timestamp_":1744670910000},{"__ext__bk_46__container_name":"unify-query","_value_":4155,"_timestamp_":1744670940000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744670970000},{"__ext__bk_46__container_name":"unify-query","_value_":4661,"_timestamp_":1744671000000},{"__ext__bk_46__container_name":"unify-query","_value_":3494,"_timestamp_":1744671030000},{"__ext__bk_46__container_name":"unify-query","_value_":4442,"_timestamp_":1744671060000},{"__ext__bk_46__container_name":"unify-query","_value_":3643,"_timestamp_":1744671090000},{"__ext__bk_46__container_name":"unify-query","_value_":4755,"_timestamp_":1744671120000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744671150000},{"__ext__bk_46__container_name":"unify-query","_value_":4284,"_timestamp_":1744671180000},{"__ext__bk_46__container_name":"unify-query","_value_":3258,"_timestamp_":1744671210000},{"__ext__bk_46__container_name":"unify-query","_value_":4453,"_timestamp_":1744671240000},{"__ext__bk_46__container_name":"unify-query","_value_":3431,"_timestamp_":1744671270000},{"__ext__bk_46__container_name":"unify-query","_value_":4231,"_timestamp_":1744671300000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744671330000},{"__ext__bk_46__container_name":"unify-query","_value_":3907,"_timestamp_":1744671360000},{"__ext__bk_46__container_name":"unify-query","_value_":3524,"_timestamp_":1744671390000},{"__ext__bk_46__container_name":"unify-query","_value_":4438,"_timestamp_":1744671420000},{"__ext__bk_46__container_name":"unify-query","_value_":3547,"_timestamp_":1744671450000},{"__ext__bk_46__container_name":"unify-query","_value_":4033,"_timestamp_":1744671480000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744671510000},{"__ext__bk_46__container_name":"unify-query","_value_":4162,"_timestamp_":1744671540000},{"__ext__bk_46__container_name":"unify-query","_value_":3588,"_timestamp_":1744671570000},{"__ext__bk_46__container_name":"unify-query","_value_":16444,"_timestamp_":1744671600000},{"__ext__bk_46__container_name":"unify-query","_value_":15396,"_timestamp_":1744671630000},{"__ext__bk_46__container_name":"unify-query","_value_":3024,"_timestamp_":1744671660000},{"__ext__bk_46__container_name":"unify-query","_value_":12656,"_timestamp_":1744671690000},{"__ext__bk_46__container_name":"unify-query","_value_":4733,"_timestamp_":1744671720000},{"__ext__bk_46__container_name":"unify-query","_value_":3766,"_timestamp_":1744671750000},{"__ext__bk_46__container_name":"unify-query","_value_":4388,"_timestamp_":1744671780000},{"__ext__bk_46__container_name":"unify-query","_value_":3340,"_timestamp_":1744671810000},{"__ext__bk_46__container_name":"unify-query","_value_":4487,"_timestamp_":1744671840000},{"__ext__bk_46__container_name":"unify-query","_value_":3549,"_timestamp_":1744671870000},{"__ext__bk_46__container_name":"unify-query","_value_":4154,"_timestamp_":1744671900000},{"__ext__bk_46__container_name":"unify-query","_value_":3406,"_timestamp_":1744671930000},{"__ext__bk_46__container_name":"unify-query","_value_":4314,"_timestamp_":1744671960000},{"__ext__bk_46__container_name":"unify-query","_value_":3472,"_timestamp_":1744671990000},{"__ext__bk_46__container_name":"unify-query","_value_":4309,"_timestamp_":1744672020000},{"__ext__bk_46__container_name":"unify-query","_value_":3458,"_timestamp_":1744672050000},{"__ext__bk_46__container_name":"unify-query","_value_":4191,"_timestamp_":1744672080000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744672110000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744672140000},{"__ext__bk_46__container_name":"unify-query","_value_":3525,"_timestamp_":1744672170000},{"__ext__bk_46__container_name":"unify-query","_value_":4445,"_timestamp_":1744672200000},{"__ext__bk_46__container_name":"unify-query","_value_":3822,"_timestamp_":1744672230000},{"__ext__bk_46__container_name":"unify-query","_value_":4346,"_timestamp_":1744672260000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744672290000},{"__ext__bk_46__container_name":"unify-query","_value_":4615,"_timestamp_":1744672320000},{"__ext__bk_46__container_name":"unify-query","_value_":3591,"_timestamp_":1744672350000},{"__ext__bk_46__container_name":"unify-query","_value_":4056,"_timestamp_":1744672380000},{"__ext__bk_46__container_name":"unify-query","_value_":3544,"_timestamp_":1744672410000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744672440000},{"__ext__bk_46__container_name":"unify-query","_value_":3647,"_timestamp_":1744672470000},{"__ext__bk_46__container_name":"unify-query","_value_":4887,"_timestamp_":1744672500000},{"__ext__bk_46__container_name":"unify-query","_value_":3450,"_timestamp_":1744672530000},{"__ext__bk_46__container_name":"unify-query","_value_":4302,"_timestamp_":1744672560000},{"__ext__bk_46__container_name":"unify-query","_value_":3425,"_timestamp_":1744672590000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744672620000},{"__ext__bk_46__container_name":"unify-query","_value_":3532,"_timestamp_":1744672650000},{"__ext__bk_46__container_name":"unify-query","_value_":4282,"_timestamp_":1744672680000},{"__ext__bk_46__container_name":"unify-query","_value_":3571,"_timestamp_":1744672710000},{"__ext__bk_46__container_name":"unify-query","_value_":4182,"_timestamp_":1744672740000},{"__ext__bk_46__container_name":"unify-query","_value_":3210,"_timestamp_":1744672770000},{"__ext__bk_46__container_name":"unify-query","_value_":6383,"_timestamp_":1744672800000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744672830000},{"__ext__bk_46__container_name":"unify-query","_value_":4408,"_timestamp_":1744672860000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744672890000},{"__ext__bk_46__container_name":"unify-query","_value_":4795,"_timestamp_":1744672920000},{"__ext__bk_46__container_name":"unify-query","_value_":3632,"_timestamp_":1744672950000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744672980000},{"__ext__bk_46__container_name":"unify-query","_value_":3534,"_timestamp_":1744673010000},{"__ext__bk_46__container_name":"unify-query","_value_":4212,"_timestamp_":1744673040000},{"__ext__bk_46__container_name":"unify-query","_value_":3380,"_timestamp_":1744673070000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744673100000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744673130000},{"__ext__bk_46__container_name":"unify-query","_value_":4120,"_timestamp_":1744673160000},{"__ext__bk_46__container_name":"unify-query","_value_":3526,"_timestamp_":1744673190000},{"__ext__bk_46__container_name":"unify-query","_value_":4200,"_timestamp_":1744673220000},{"__ext__bk_46__container_name":"unify-query","_value_":3302,"_timestamp_":1744673250000},{"__ext__bk_46__container_name":"unify-query","_value_":4370,"_timestamp_":1744673280000},{"__ext__bk_46__container_name":"unify-query","_value_":3462,"_timestamp_":1744673310000},{"__ext__bk_46__container_name":"unify-query","_value_":4223,"_timestamp_":1744673340000},{"__ext__bk_46__container_name":"unify-query","_value_":3564,"_timestamp_":1744673370000},{"__ext__bk_46__container_name":"unify-query","_value_":12072,"_timestamp_":1744673400000},{"__ext__bk_46__container_name":"unify-query","_value_":17986,"_timestamp_":1744673430000},{"__ext__bk_46__container_name":"unify-query","_value_":4089,"_timestamp_":1744673460000},{"__ext__bk_46__container_name":"unify-query","_value_":12000,"_timestamp_":1744673490000},{"__ext__bk_46__container_name":"unify-query","_value_":4790,"_timestamp_":1744673520000},{"__ext__bk_46__container_name":"unify-query","_value_":3637,"_timestamp_":1744673550000},{"__ext__bk_46__container_name":"unify-query","_value_":4177,"_timestamp_":1744673580000},{"__ext__bk_46__container_name":"unify-query","_value_":3438,"_timestamp_":1744673610000},{"__ext__bk_46__container_name":"unify-query","_value_":4465,"_timestamp_":1744673640000},{"__ext__bk_46__container_name":"unify-query","_value_":3627,"_timestamp_":1744673670000},{"__ext__bk_46__container_name":"unify-query","_value_":4131,"_timestamp_":1744673700000},{"__ext__bk_46__container_name":"unify-query","_value_":3396,"_timestamp_":1744673730000},{"__ext__bk_46__container_name":"unify-query","_value_":4395,"_timestamp_":1744673760000},{"__ext__bk_46__container_name":"unify-query","_value_":3638,"_timestamp_":1744673790000},{"__ext__bk_46__container_name":"unify-query","_value_":4093,"_timestamp_":1744673820000},{"__ext__bk_46__container_name":"unify-query","_value_":3584,"_timestamp_":1744673850000},{"__ext__bk_46__container_name":"unify-query","_value_":4082,"_timestamp_":1744673880000},{"__ext__bk_46__container_name":"unify-query","_value_":3475,"_timestamp_":1744673910000},{"__ext__bk_46__container_name":"unify-query","_value_":4051,"_timestamp_":1744673940000},{"__ext__bk_46__container_name":"unify-query","_value_":3354,"_timestamp_":1744673970000},{"__ext__bk_46__container_name":"unify-query","_value_":6296,"_timestamp_":1744674000000},{"__ext__bk_46__container_name":"unify-query","_value_":3473,"_timestamp_":1744674030000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744674060000},{"__ext__bk_46__container_name":"unify-query","_value_":3793,"_timestamp_":1744674090000},{"__ext__bk_46__container_name":"unify-query","_value_":4391,"_timestamp_":1744674120000},{"__ext__bk_46__container_name":"unify-query","_value_":3836,"_timestamp_":1744674150000},{"__ext__bk_46__container_name":"unify-query","_value_":4190,"_timestamp_":1744674180000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674210000},{"__ext__bk_46__container_name":"unify-query","_value_":4230,"_timestamp_":1744674240000},{"__ext__bk_46__container_name":"unify-query","_value_":3488,"_timestamp_":1744674270000},{"__ext__bk_46__container_name":"unify-query","_value_":4964,"_timestamp_":1744674300000},{"__ext__bk_46__container_name":"unify-query","_value_":3455,"_timestamp_":1744674330000},{"__ext__bk_46__container_name":"unify-query","_value_":4116,"_timestamp_":1744674360000},{"__ext__bk_46__container_name":"unify-query","_value_":3250,"_timestamp_":1744674390000},{"__ext__bk_46__container_name":"unify-query","_value_":4494,"_timestamp_":1744674420000},{"__ext__bk_46__container_name":"unify-query","_value_":3326,"_timestamp_":1744674450000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744674480000},{"__ext__bk_46__container_name":"unify-query","_value_":3580,"_timestamp_":1744674510000},{"__ext__bk_46__container_name":"unify-query","_value_":4368,"_timestamp_":1744674540000},{"__ext__bk_46__container_name":"unify-query","_value_":3685,"_timestamp_":1744674570000},{"__ext__bk_46__container_name":"unify-query","_value_":4381,"_timestamp_":1744674600000},{"__ext__bk_46__container_name":"unify-query","_value_":3699,"_timestamp_":1744674630000},{"__ext__bk_46__container_name":"unify-query","_value_":4513,"_timestamp_":1744674660000},{"__ext__bk_46__container_name":"unify-query","_value_":3729,"_timestamp_":1744674690000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744674720000},{"__ext__bk_46__container_name":"unify-query","_value_":3639,"_timestamp_":1744674750000},{"__ext__bk_46__container_name":"unify-query","_value_":4018,"_timestamp_":1744674780000},{"__ext__bk_46__container_name":"unify-query","_value_":3587,"_timestamp_":1744674810000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744674840000},{"__ext__bk_46__container_name":"unify-query","_value_":3389,"_timestamp_":1744674870000},{"__ext__bk_46__container_name":"unify-query","_value_":4289,"_timestamp_":1744674900000},{"__ext__bk_46__container_name":"unify-query","_value_":3540,"_timestamp_":1744674930000},{"__ext__bk_46__container_name":"unify-query","_value_":4106,"_timestamp_":1744674960000},{"__ext__bk_46__container_name":"unify-query","_value_":3478,"_timestamp_":1744674990000},{"__ext__bk_46__container_name":"unify-query","_value_":4268,"_timestamp_":1744675020000},{"__ext__bk_46__container_name":"unify-query","_value_":3577,"_timestamp_":1744675050000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744675080000},{"__ext__bk_46__container_name":"unify-query","_value_":3511,"_timestamp_":1744675110000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744675140000},{"__ext__bk_46__container_name":"unify-query","_value_":3573,"_timestamp_":1744675170000},{"__ext__bk_46__container_name":"unify-query","_value_":17095,"_timestamp_":1744675200000},{"__ext__bk_46__container_name":"unify-query","_value_":14907,"_timestamp_":1744675230000},{"__ext__bk_46__container_name":"unify-query","_value_":6455,"_timestamp_":1744675260000},{"__ext__bk_46__container_name":"unify-query","_value_":9818,"_timestamp_":1744675290000},{"__ext__bk_46__container_name":"unify-query","_value_":5253,"_timestamp_":1744675320000},{"__ext__bk_46__container_name":"unify-query","_value_":3567,"_timestamp_":1744675350000},{"__ext__bk_46__container_name":"unify-query","_value_":4047,"_timestamp_":1744675380000},{"__ext__bk_46__container_name":"unify-query","_value_":3342,"_timestamp_":1744675410000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744675440000},{"__ext__bk_46__container_name":"unify-query","_value_":3394,"_timestamp_":1744675470000},{"__ext__bk_46__container_name":"unify-query","_value_":4260,"_timestamp_":1744675500000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744675530000},{"__ext__bk_46__container_name":"unify-query","_value_":4341,"_timestamp_":1744675560000},{"__ext__bk_46__container_name":"unify-query","_value_":3559,"_timestamp_":1744675590000},{"__ext__bk_46__container_name":"unify-query","_value_":4188,"_timestamp_":1744675620000},{"__ext__bk_46__container_name":"unify-query","_value_":3519,"_timestamp_":1744675650000},{"__ext__bk_46__container_name":"unify-query","_value_":4143,"_timestamp_":1744675680000},{"__ext__bk_46__container_name":"unify-query","_value_":3630,"_timestamp_":1744675710000},{"__ext__bk_46__container_name":"unify-query","_value_":4042,"_timestamp_":1744675740000},{"__ext__bk_46__container_name":"unify-query","_value_":3653,"_timestamp_":1744675770000},{"__ext__bk_46__container_name":"unify-query","_value_":4358,"_timestamp_":1744675800000},{"__ext__bk_46__container_name":"unify-query","_value_":3688,"_timestamp_":1744675830000},{"__ext__bk_46__container_name":"unify-query","_value_":4450,"_timestamp_":1744675860000},{"__ext__bk_46__container_name":"unify-query","_value_":3387,"_timestamp_":1744675890000},{"__ext__bk_46__container_name":"unify-query","_value_":4864,"_timestamp_":1744675920000},{"__ext__bk_46__container_name":"unify-query","_value_":3629,"_timestamp_":1744675950000},{"__ext__bk_46__container_name":"unify-query","_value_":4127,"_timestamp_":1744675980000},{"__ext__bk_46__container_name":"unify-query","_value_":3424,"_timestamp_":1744676010000},{"__ext__bk_46__container_name":"unify-query","_value_":4267,"_timestamp_":1744676040000},{"__ext__bk_46__container_name":"unify-query","_value_":3328,"_timestamp_":1744676070000},{"__ext__bk_46__container_name":"unify-query","_value_":5128,"_timestamp_":1744676100000},{"__ext__bk_46__container_name":"unify-query","_value_":3657,"_timestamp_":1744676130000},{"__ext__bk_46__container_name":"unify-query","_value_":4185,"_timestamp_":1744676160000},{"__ext__bk_46__container_name":"unify-query","_value_":3336,"_timestamp_":1744676190000},{"__ext__bk_46__container_name":"unify-query","_value_":4532,"_timestamp_":1744676220000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744676250000},{"__ext__bk_46__container_name":"unify-query","_value_":4174,"_timestamp_":1744676280000},{"__ext__bk_46__container_name":"unify-query","_value_":3318,"_timestamp_":1744676310000},{"__ext__bk_46__container_name":"unify-query","_value_":4463,"_timestamp_":1744676340000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676370000},{"__ext__bk_46__container_name":"unify-query","_value_":6064,"_timestamp_":1744676400000},{"__ext__bk_46__container_name":"unify-query","_value_":3292,"_timestamp_":1744676430000},{"__ext__bk_46__container_name":"unify-query","_value_":4858,"_timestamp_":1744676460000},{"__ext__bk_46__container_name":"unify-query","_value_":3543,"_timestamp_":1744676490000},{"__ext__bk_46__container_name":"unify-query","_value_":4620,"_timestamp_":1744676520000},{"__ext__bk_46__container_name":"unify-query","_value_":3750,"_timestamp_":1744676550000},{"__ext__bk_46__container_name":"unify-query","_value_":4043,"_timestamp_":1744676580000},{"__ext__bk_46__container_name":"unify-query","_value_":3595,"_timestamp_":1744676610000},{"__ext__bk_46__container_name":"unify-query","_value_":4152,"_timestamp_":1744676640000},{"__ext__bk_46__container_name":"unify-query","_value_":3550,"_timestamp_":1744676670000},{"__ext__bk_46__container_name":"unify-query","_value_":4011,"_timestamp_":1744676700000},{"__ext__bk_46__container_name":"unify-query","_value_":3502,"_timestamp_":1744676730000},{"__ext__bk_46__container_name":"unify-query","_value_":4050,"_timestamp_":1744676760000},{"__ext__bk_46__container_name":"unify-query","_value_":3118,"_timestamp_":1744676790000},{"__ext__bk_46__container_name":"unify-query","_value_":4628,"_timestamp_":1744676820000},{"__ext__bk_46__container_name":"unify-query","_value_":3441,"_timestamp_":1744676850000},{"__ext__bk_46__container_name":"unify-query","_value_":4366,"_timestamp_":1744676880000},{"__ext__bk_46__container_name":"unify-query","_value_":3500,"_timestamp_":1744676910000},{"__ext__bk_46__container_name":"unify-query","_value_":4160,"_timestamp_":1744676940000},{"__ext__bk_46__container_name":"unify-query","_value_":3662,"_timestamp_":1744676970000},{"__ext__bk_46__container_name":"unify-query","_value_":11392,"_timestamp_":1744677000000},{"__ext__bk_46__container_name":"unify-query","_value_":18649,"_timestamp_":1744677030000},{"__ext__bk_46__container_name":"unify-query","_value_":7107,"_timestamp_":1744677060000},{"__ext__bk_46__container_name":"unify-query","_value_":9213,"_timestamp_":1744677090000},{"__ext__bk_46__container_name":"unify-query","_value_":4235,"_timestamp_":1744677120000},{"__ext__bk_46__container_name":"unify-query","_value_":3623,"_timestamp_":1744677150000},{"__ext__bk_46__container_name":"unify-query","_value_":4412,"_timestamp_":1744677180000},{"__ext__bk_46__container_name":"unify-query","_value_":3436,"_timestamp_":1744677210000},{"__ext__bk_46__container_name":"unify-query","_value_":4233,"_timestamp_":1744677240000},{"__ext__bk_46__container_name":"unify-query","_value_":3440,"_timestamp_":1744677270000},{"__ext__bk_46__container_name":"unify-query","_value_":4383,"_timestamp_":1744677300000},{"__ext__bk_46__container_name":"unify-query","_value_":3507,"_timestamp_":1744677330000},{"__ext__bk_46__container_name":"unify-query","_value_":4288,"_timestamp_":1744677360000},{"__ext__bk_46__container_name":"unify-query","_value_":3197,"_timestamp_":1744677390000},{"__ext__bk_46__container_name":"unify-query","_value_":4605,"_timestamp_":1744677420000},{"__ext__bk_46__container_name":"unify-query","_value_":3249,"_timestamp_":1744677450000},{"__ext__bk_46__container_name":"unify-query","_value_":4421,"_timestamp_":1744677480000},{"__ext__bk_46__container_name":"unify-query","_value_":2998,"_timestamp_":1744677510000},{"__ext__bk_46__container_name":"unify-query","_value_":4700,"_timestamp_":1744677540000},{"__ext__bk_46__container_name":"unify-query","_value_":3598,"_timestamp_":1744677570000},{"__ext__bk_46__container_name":"unify-query","_value_":5781,"_timestamp_":1744677600000},{"__ext__bk_46__container_name":"unify-query","_value_":3734,"_timestamp_":1744677630000},{"__ext__bk_46__container_name":"unify-query","_value_":4510,"_timestamp_":1744677660000},{"__ext__bk_46__container_name":"unify-query","_value_":3752,"_timestamp_":1744677690000},{"__ext__bk_46__container_name":"unify-query","_value_":4447,"_timestamp_":1744677720000},{"__ext__bk_46__container_name":"unify-query","_value_":3523,"_timestamp_":1744677750000},{"__ext__bk_46__container_name":"unify-query","_value_":4187,"_timestamp_":1744677780000},{"__ext__bk_46__container_name":"unify-query","_value_":3640,"_timestamp_":1744677810000},{"__ext__bk_46__container_name":"unify-query","_value_":3900,"_timestamp_":1744677840000},{"__ext__bk_46__container_name":"unify-query","_value_":3514,"_timestamp_":1744677870000},{"__ext__bk_46__container_name":"unify-query","_value_":4863,"_timestamp_":1744677900000},{"__ext__bk_46__container_name":"unify-query","_value_":3565,"_timestamp_":1744677930000},{"__ext__bk_46__container_name":"unify-query","_value_":4335,"_timestamp_":1744677960000},{"__ext__bk_46__container_name":"unify-query","_value_":3533,"_timestamp_":1744677990000},{"__ext__bk_46__container_name":"unify-query","_value_":4307,"_timestamp_":1744678020000},{"__ext__bk_46__container_name":"unify-query","_value_":3556,"_timestamp_":1744678050000},{"__ext__bk_46__container_name":"unify-query","_value_":4179,"_timestamp_":1744678080000},{"__ext__bk_46__container_name":"unify-query","_value_":3664,"_timestamp_":1744678110000},{"__ext__bk_46__container_name":"unify-query","_value_":4362,"_timestamp_":1744678140000},{"__ext__bk_46__container_name":"unify-query","_value_":3222,"_timestamp_":1744678170000},{"__ext__bk_46__container_name":"unify-query","_value_":4750,"_timestamp_":1744678200000},{"__ext__bk_46__container_name":"unify-query","_value_":3546,"_timestamp_":1744678230000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744678260000},{"__ext__bk_46__container_name":"unify-query","_value_":3702,"_timestamp_":1744678290000},{"__ext__bk_46__container_name":"unify-query","_value_":4564,"_timestamp_":1744678320000},{"__ext__bk_46__container_name":"unify-query","_value_":3610,"_timestamp_":1744678350000},{"__ext__bk_46__container_name":"unify-query","_value_":4130,"_timestamp_":1744678380000},{"__ext__bk_46__container_name":"unify-query","_value_":3412,"_timestamp_":1744678410000},{"__ext__bk_46__container_name":"unify-query","_value_":4614,"_timestamp_":1744678440000},{"__ext__bk_46__container_name":"unify-query","_value_":3522,"_timestamp_":1744678470000},{"__ext__bk_46__container_name":"unify-query","_value_":4148,"_timestamp_":1744678500000},{"__ext__bk_46__container_name":"unify-query","_value_":3408,"_timestamp_":1744678530000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744678560000},{"__ext__bk_46__container_name":"unify-query","_value_":3607,"_timestamp_":1744678590000},{"__ext__bk_46__container_name":"unify-query","_value_":4172,"_timestamp_":1744678620000},{"__ext__bk_46__container_name":"unify-query","_value_":3529,"_timestamp_":1744678650000},{"__ext__bk_46__container_name":"unify-query","_value_":4227,"_timestamp_":1744678680000},{"__ext__bk_46__container_name":"unify-query","_value_":3487,"_timestamp_":1744678710000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744678740000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744678770000},{"__ext__bk_46__container_name":"unify-query","_value_":7230,"_timestamp_":1744678800000},{"__ext__bk_46__container_name":"unify-query","_value_":3818,"_timestamp_":1744678830000},{"__ext__bk_46__container_name":"unify-query","_value_":11924,"_timestamp_":1744678860000},{"__ext__bk_46__container_name":"unify-query","_value_":27269,"_timestamp_":1744678890000},{"__ext__bk_46__container_name":"unify-query","_value_":5073,"_timestamp_":1744678920000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744678950000},{"__ext__bk_46__container_name":"unify-query","_value_":4474,"_timestamp_":1744678980000},{"__ext__bk_46__container_name":"unify-query","_value_":3536,"_timestamp_":1744679010000},{"__ext__bk_46__container_name":"unify-query","_value_":4525,"_timestamp_":1744679040000},{"__ext__bk_46__container_name":"unify-query","_value_":3503,"_timestamp_":1744679070000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744679100000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744679130000},{"__ext__bk_46__container_name":"unify-query","_value_":4259,"_timestamp_":1744679160000},{"__ext__bk_46__container_name":"unify-query","_value_":3611,"_timestamp_":1744679190000},{"__ext__bk_46__container_name":"unify-query","_value_":4218,"_timestamp_":1744679220000},{"__ext__bk_46__container_name":"unify-query","_value_":3622,"_timestamp_":1744679250000},{"__ext__bk_46__container_name":"unify-query","_value_":4417,"_timestamp_":1744679280000},{"__ext__bk_46__container_name":"unify-query","_value_":3730,"_timestamp_":1744679310000},{"__ext__bk_46__container_name":"unify-query","_value_":4204,"_timestamp_":1744679340000},{"__ext__bk_46__container_name":"unify-query","_value_":3641,"_timestamp_":1744679370000},{"__ext__bk_46__container_name":"unify-query","_value_":4849,"_timestamp_":1744679400000},{"__ext__bk_46__container_name":"unify-query","_value_":3803,"_timestamp_":1744679430000},{"__ext__bk_46__container_name":"unify-query","_value_":4398,"_timestamp_":1744679460000},{"__ext__bk_46__container_name":"unify-query","_value_":3674,"_timestamp_":1744679490000},{"__ext__bk_46__container_name":"unify-query","_value_":4727,"_timestamp_":1744679520000},{"__ext__bk_46__container_name":"unify-query","_value_":3926,"_timestamp_":1744679550000},{"__ext__bk_46__container_name":"unify-query","_value_":4173,"_timestamp_":1744679580000},{"__ext__bk_46__container_name":"unify-query","_value_":3531,"_timestamp_":1744679610000},{"__ext__bk_46__container_name":"unify-query","_value_":4968,"_timestamp_":1744679640000},{"__ext__bk_46__container_name":"unify-query","_value_":3432,"_timestamp_":1744679670000},{"__ext__bk_46__container_name":"unify-query","_value_":5059,"_timestamp_":1744679700000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744679730000},{"__ext__bk_46__container_name":"unify-query","_value_":4087,"_timestamp_":1744679760000},{"__ext__bk_46__container_name":"unify-query","_value_":3590,"_timestamp_":1744679790000},{"__ext__bk_46__container_name":"unify-query","_value_":4436,"_timestamp_":1744679820000},{"__ext__bk_46__container_name":"unify-query","_value_":5299,"_timestamp_":1744679850000},{"__ext__bk_46__container_name":"unify-query","_value_":4320,"_timestamp_":1744679880000},{"__ext__bk_46__container_name":"unify-query","_value_":3861,"_timestamp_":1744679910000},{"__ext__bk_46__container_name":"unify-query","_value_":4511,"_timestamp_":1744679940000},{"__ext__bk_46__container_name":"unify-query","_value_":3711,"_timestamp_":1744679970000},{"__ext__bk_46__container_name":"unify-query","_value_":6021,"_timestamp_":1744680000000},{"__ext__bk_46__container_name":"unify-query","_value_":3942,"_timestamp_":1744680030000},{"__ext__bk_46__container_name":"unify-query","_value_":4800,"_timestamp_":1744680060000},{"__ext__bk_46__container_name":"unify-query","_value_":3681,"_timestamp_":1744680090000},{"__ext__bk_46__container_name":"unify-query","_value_":4592,"_timestamp_":1744680120000},{"__ext__bk_46__container_name":"unify-query","_value_":3560,"_timestamp_":1744680150000},{"__ext__bk_46__container_name":"unify-query","_value_":4194,"_timestamp_":1744680180000},{"__ext__bk_46__container_name":"unify-query","_value_":3490,"_timestamp_":1744680210000},{"__ext__bk_46__container_name":"unify-query","_value_":4971,"_timestamp_":1744680240000},{"__ext__bk_46__container_name":"unify-query","_value_":4009,"_timestamp_":1744680270000},{"__ext__bk_46__container_name":"unify-query","_value_":4837,"_timestamp_":1744680300000},{"__ext__bk_46__container_name":"unify-query","_value_":3227,"_timestamp_":1744680330000},{"__ext__bk_46__container_name":"unify-query","_value_":4531,"_timestamp_":1744680360000},{"__ext__bk_46__container_name":"unify-query","_value_":2888,"_timestamp_":1744680390000},{"__ext__bk_46__container_name":"unify-query","_value_":5083,"_timestamp_":1744680420000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744680450000},{"__ext__bk_46__container_name":"unify-query","_value_":4207,"_timestamp_":1744680480000},{"__ext__bk_46__container_name":"unify-query","_value_":3373,"_timestamp_":1744680510000},{"__ext__bk_46__container_name":"unify-query","_value_":4482,"_timestamp_":1744680540000},{"__ext__bk_46__container_name":"unify-query","_value_":3110,"_timestamp_":1744680570000},{"__ext__bk_46__container_name":"unify-query","_value_":13551,"_timestamp_":1744680600000},{"__ext__bk_46__container_name":"unify-query","_value_":17159,"_timestamp_":1744680630000},{"__ext__bk_46__container_name":"unify-query","_value_":6284,"_timestamp_":1744680660000},{"__ext__bk_46__container_name":"unify-query","_value_":9924,"_timestamp_":1744680690000},{"__ext__bk_46__container_name":"unify-query","_value_":4547,"_timestamp_":1744680720000},{"__ext__bk_46__container_name":"unify-query","_value_":3474,"_timestamp_":1744680750000},{"__ext__bk_46__container_name":"unify-query","_value_":4312,"_timestamp_":1744680780000},{"__ext__bk_46__container_name":"unify-query","_value_":3689,"_timestamp_":1744680810000},{"__ext__bk_46__container_name":"unify-query","_value_":4680,"_timestamp_":1744680840000},{"__ext__bk_46__container_name":"unify-query","_value_":3609,"_timestamp_":1744680870000},{"__ext__bk_46__container_name":"unify-query","_value_":4886,"_timestamp_":1744680900000},{"__ext__bk_46__container_name":"unify-query","_value_":3842,"_timestamp_":1744680930000},{"__ext__bk_46__container_name":"unify-query","_value_":4810,"_timestamp_":1744680960000},{"__ext__bk_46__container_name":"unify-query","_value_":4102,"_timestamp_":1744680990000},{"__ext__bk_46__container_name":"unify-query","_value_":4594,"_timestamp_":1744681020000},{"__ext__bk_46__container_name":"unify-query","_value_":4168,"_timestamp_":1744681050000},{"__ext__bk_46__container_name":"unify-query","_value_":4562,"_timestamp_":1744681080000},{"__ext__bk_46__container_name":"unify-query","_value_":4506,"_timestamp_":1744681110000},{"__ext__bk_46__container_name":"unify-query","_value_":5243,"_timestamp_":1744681140000},{"__ext__bk_46__container_name":"unify-query","_value_":5135,"_timestamp_":1744681170000},{"__ext__bk_46__container_name":"unify-query","_value_":6671,"_timestamp_":1744681200000},{"__ext__bk_46__container_name":"unify-query","_value_":3806,"_timestamp_":1744681230000},{"__ext__bk_46__container_name":"unify-query","_value_":4535,"_timestamp_":1744681260000},{"__ext__bk_46__container_name":"unify-query","_value_":3721,"_timestamp_":1744681290000},{"__ext__bk_46__container_name":"unify-query","_value_":4799,"_timestamp_":1744681320000},{"__ext__bk_46__container_name":"unify-query","_value_":3909,"_timestamp_":1744681350000},{"__ext__bk_46__container_name":"unify-query","_value_":4261,"_timestamp_":1744681380000},{"__ext__bk_46__container_name":"unify-query","_value_":3671,"_timestamp_":1744681410000},{"__ext__bk_46__container_name":"unify-query","_value_":4359,"_timestamp_":1744681440000},{"__ext__bk_46__container_name":"unify-query","_value_":4063,"_timestamp_":1744681470000},{"__ext__bk_46__container_name":"unify-query","_value_":5231,"_timestamp_":1744681500000},{"__ext__bk_46__container_name":"unify-query","_value_":3778,"_timestamp_":1744681530000},{"__ext__bk_46__container_name":"unify-query","_value_":4684,"_timestamp_":1744681560000},{"__ext__bk_46__container_name":"unify-query","_value_":4072,"_timestamp_":1744681590000},{"__ext__bk_46__container_name":"unify-query","_value_":5029,"_timestamp_":1744681620000},{"__ext__bk_46__container_name":"unify-query","_value_":3700,"_timestamp_":1744681650000},{"__ext__bk_46__container_name":"unify-query","_value_":4670,"_timestamp_":1744681680000},{"__ext__bk_46__container_name":"unify-query","_value_":3557,"_timestamp_":1744681710000},{"__ext__bk_46__container_name":"unify-query","_value_":4590,"_timestamp_":1744681740000},{"__ext__bk_46__container_name":"unify-query","_value_":3041,"_timestamp_":1744681770000},{"__ext__bk_46__container_name":"unify-query","_value_":5043,"_timestamp_":1744681800000},{"__ext__bk_46__container_name":"unify-query","_value_":3530,"_timestamp_":1744681830000},{"__ext__bk_46__container_name":"unify-query","_value_":6807,"_timestamp_":1744681860000},{"__ext__bk_46__container_name":"unify-query","_value_":4455,"_timestamp_":1744681890000},{"__ext__bk_46__container_name":"unify-query","_value_":6841,"_timestamp_":1744681920000},{"__ext__bk_46__container_name":"unify-query","_value_":4519,"_timestamp_":1744681950000},{"__ext__bk_46__container_name":"unify-query","_value_":6617,"_timestamp_":1744681980000},{"__ext__bk_46__container_name":"unify-query","_value_":4633,"_timestamp_":1744682010000},{"__ext__bk_46__container_name":"unify-query","_value_":5997,"_timestamp_":1744682040000},{"__ext__bk_46__container_name":"unify-query","_value_":4446,"_timestamp_":1744682070000},{"__ext__bk_46__container_name":"unify-query","_value_":5569,"_timestamp_":1744682100000},{"__ext__bk_46__container_name":"unify-query","_value_":4324,"_timestamp_":1744682130000},{"__ext__bk_46__container_name":"unify-query","_value_":5354,"_timestamp_":1744682160000},{"__ext__bk_46__container_name":"unify-query","_value_":7245,"_timestamp_":1744682190000},{"__ext__bk_46__container_name":"unify-query","_value_":5258,"_timestamp_":1744682220000},{"__ext__bk_46__container_name":"unify-query","_value_":4296,"_timestamp_":1744682250000},{"__ext__bk_46__container_name":"unify-query","_value_":5349,"_timestamp_":1744682280000},{"__ext__bk_46__container_name":"unify-query","_value_":4479,"_timestamp_":1744682310000},{"__ext__bk_46__container_name":"unify-query","_value_":5127,"_timestamp_":1744682340000},{"__ext__bk_46__container_name":"unify-query","_value_":4006,"_timestamp_":1744682370000},{"__ext__bk_46__container_name":"unify-query","_value_":19058,"_timestamp_":1744682400000},{"__ext__bk_46__container_name":"unify-query","_value_":14501,"_timestamp_":1744682430000},{"__ext__bk_46__container_name":"unify-query","_value_":3810,"_timestamp_":1744682460000},{"__ext__bk_46__container_name":"unify-query","_value_":12368,"_timestamp_":1744682490000},{"__ext__bk_46__container_name":"unify-query","_value_":6976,"_timestamp_":1744682520000},{"__ext__bk_46__container_name":"unify-query","_value_":4399,"_timestamp_":1744682550000},{"__ext__bk_46__container_name":"unify-query","_value_":5482,"_timestamp_":1744682580000},{"__ext__bk_46__container_name":"unify-query","_value_":4524,"_timestamp_":1744682610000},{"__ext__bk_46__container_name":"unify-query","_value_":5478,"_timestamp_":1744682640000},{"__ext__bk_46__container_name":"unify-query","_value_":4920,"_timestamp_":1744682670000},{"__ext__bk_46__container_name":"unify-query","_value_":5347,"_timestamp_":1744682700000},{"__ext__bk_46__container_name":"unify-query","_value_":4427,"_timestamp_":1744682730000},{"__ext__bk_46__container_name":"unify-query","_value_":5102,"_timestamp_":1744682760000},{"__ext__bk_46__container_name":"unify-query","_value_":4441,"_timestamp_":1744682790000},{"__ext__bk_46__container_name":"unify-query","_value_":5596,"_timestamp_":1744682820000},{"__ext__bk_46__container_name":"unify-query","_value_":4888,"_timestamp_":1744682850000},{"__ext__bk_46__container_name":"unify-query","_value_":5306,"_timestamp_":1744682880000},{"__ext__bk_46__container_name":"unify-query","_value_":4825,"_timestamp_":1744682910000},{"__ext__bk_46__container_name":"unify-query","_value_":5897,"_timestamp_":1744682940000},{"__ext__bk_46__container_name":"unify-query","_value_":4481,"_timestamp_":1744682970000},{"__ext__bk_46__container_name":"unify-query","_value_":6086,"_timestamp_":1744683000000},{"__ext__bk_46__container_name":"unify-query","_value_":4910,"_timestamp_":1744683030000},{"__ext__bk_46__container_name":"unify-query","_value_":5676,"_timestamp_":1744683060000},{"__ext__bk_46__container_name":"unify-query","_value_":3626,"_timestamp_":1744683090000},{"__ext__bk_46__container_name":"unify-query","_value_":6929,"_timestamp_":1744683120000},{"__ext__bk_46__container_name":"unify-query","_value_":4601,"_timestamp_":1744683150000},{"__ext__bk_46__container_name":"unify-query","_value_":5525,"_timestamp_":1744683180000},{"__ext__bk_46__container_name":"unify-query","_value_":4500,"_timestamp_":1744683210000},{"__ext__bk_46__container_name":"unify-query","_value_":5617,"_timestamp_":1744683240000},{"__ext__bk_46__container_name":"unify-query","_value_":4503,"_timestamp_":1744683270000},{"__ext__bk_46__container_name":"unify-query","_value_":6328,"_timestamp_":1744683300000},{"__ext__bk_46__container_name":"unify-query","_value_":4557,"_timestamp_":1744683330000},{"__ext__bk_46__container_name":"unify-query","_value_":5356,"_timestamp_":1744683360000},{"__ext__bk_46__container_name":"unify-query","_value_":4413,"_timestamp_":1744683390000},{"__ext__bk_46__container_name":"unify-query","_value_":5335,"_timestamp_":1744683420000},{"__ext__bk_46__container_name":"unify-query","_value_":4640,"_timestamp_":1744683450000},{"__ext__bk_46__container_name":"unify-query","_value_":5399,"_timestamp_":1744683480000},{"__ext__bk_46__container_name":"unify-query","_value_":4298,"_timestamp_":1744683510000},{"__ext__bk_46__container_name":"unify-query","_value_":5415,"_timestamp_":1744683540000},{"__ext__bk_46__container_name":"unify-query","_value_":4540,"_timestamp_":1744683570000},{"__ext__bk_46__container_name":"unify-query","_value_":6949,"_timestamp_":1744683600000},{"__ext__bk_46__container_name":"unify-query","_value_":4574,"_timestamp_":1744683630000},{"__ext__bk_46__container_name":"unify-query","_value_":5757,"_timestamp_":1744683660000},{"__ext__bk_46__container_name":"unify-query","_value_":4669,"_timestamp_":1744683690000},{"__ext__bk_46__container_name":"unify-query","_value_":5706,"_timestamp_":1744683720000},{"__ext__bk_46__container_name":"unify-query","_value_":4472,"_timestamp_":1744683750000},{"__ext__bk_46__container_name":"unify-query","_value_":5386,"_timestamp_":1744683780000},{"__ext__bk_46__container_name":"unify-query","_value_":4490,"_timestamp_":1744683810000},{"__ext__bk_46__container_name":"unify-query","_value_":5104,"_timestamp_":1744683840000},{"__ext__bk_46__container_name":"unify-query","_value_":4201,"_timestamp_":1744683870000},{"__ext__bk_46__container_name":"unify-query","_value_":5979,"_timestamp_":1744683900000},{"__ext__bk_46__container_name":"unify-query","_value_":4853,"_timestamp_":1744683930000},{"__ext__bk_46__container_name":"unify-query","_value_":6691,"_timestamp_":1744683960000},{"__ext__bk_46__container_name":"unify-query","_value_":4572,"_timestamp_":1744683990000},{"__ext__bk_46__container_name":"unify-query","_value_":5554,"_timestamp_":1744684020000},{"__ext__bk_46__container_name":"unify-query","_value_":5244,"_timestamp_":1744684050000},{"__ext__bk_46__container_name":"unify-query","_value_":5392,"_timestamp_":1744684080000},{"__ext__bk_46__container_name":"unify-query","_value_":4550,"_timestamp_":1744684110000},{"__ext__bk_46__container_name":"unify-query","_value_":520,"_timestamp_":1744684140000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":52,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":8,"connect_db":55,"match_query_routing_rule":0,"check_permission":73,"check_query_semantic":0,"pick_valid_storage":1},"total_record_size":269248,"timetaken":0.191,"result_schema":[{"field_type":"string","field_name":"__c0","field_alias":"__ext__bk_46__container_name","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1},{"field_type":"long","field_name":"__c2","field_alias":"_timestamp_","field_index":2}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, }) for i, c := range map[string]struct { diff --git a/pkg/unify-query/tsdb/bksql/format_test.go b/pkg/unify-query/tsdb/bksql/format_test.go index 21d89dabf..144650b98 100644 --- a/pkg/unify-query/tsdb/bksql/format_test.go +++ b/pkg/unify-query/tsdb/bksql/format_test.go @@ -243,7 +243,7 @@ func TestNewSqlFactory(t *testing.T) { }, }, }, - expected: "SELECT CAST(__ext[\"container_id\"] AS STRING) AS `__ext__bk_46__container_id`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY CAST(__ext[\"container_id\"] AS STRING), _timestamp_ ORDER BY `_timestamp_` ASC", + expected: "SELECT CAST(__ext[\"container_id\"] AS STRING) AS `__ext__bk_46__container_id`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1440 AS INT) * 1440 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741935945000 AND `dtEventTimeStamp` <= 1742456145000 AND `thedate` >= '20250314' AND `thedate` <= '20250320' GROUP BY __ext__bk_46__container_id, _timestamp_ ORDER BY `_timestamp_` ASC", }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 4771bd553..af11f5350 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -917,7 +917,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY __ext__bk_46__io_kubernetes_workload_name, __ext__bk_46__io_kubernetes_workload_type ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC LIMIT 3", }, // 测试用例10: doris 处理 object 字段 + 时间聚合 { @@ -956,7 +956,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, CAST(__ext[\"io_kubernetes_workload_type\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_type`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' AND CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) = 'bkm-daemonset-worker' AND `bk_host_id` = '267730' GROUP BY __ext__bk_46__io_kubernetes_workload_name, __ext__bk_46__io_kubernetes_workload_type, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例11: doris 处理 object 字段 + 时间聚合 5m { @@ -982,7 +982,7 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 5 AS INT) * 5 - 0) * 60 * 1000) AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY __ext__bk_46__io_kubernetes_workload_name, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", }, // 测试用例12: doris 处理 object 字段 + 时间聚合 15s { @@ -1008,7 +1008,41 @@ func TestInstance_bkSql_EdgeCases(t *testing.T) { }, start: time.Unix(1741334700, 0), end: time.Unix(1741335000, 0), - expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING), _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + expected: "SELECT CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) AS `__ext__bk_46__io_kubernetes_workload_name`, COUNT(CAST(__ext[\"container_id\"] AS STRING)) AS `_value_`, CAST(dtEventTimeStamp / 15000 AS INT) * 15000 AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1741334700000 AND `dtEventTimeStamp` <= 1741335000000 AND `thedate` = '20250307' GROUP BY __ext__bk_46__io_kubernetes_workload_name, _timestamp_ ORDER BY CAST(__ext[\"io_kubernetes_workload_name\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 3", + }, + // 测试用例13: doris 处理多层级 object 字段 + { + name: "doris default multiple order fields and time aggregate 1m", + query: &metadata.Query{ + DB: "2_bkapm_trace_bkop_doris", + Measurement: sqlExpr.Doris, + Field: "attributes.http.host", + Size: 1, + AllConditions: metadata.AllConditions{ + { + { + DimensionName: "attributes.http.host", + Operator: metadata.ConditionNotEqual, + Value: []string{""}, + }, + }, + }, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Dimensions: []string{"attributes.http.host"}, + Window: time.Minute, + }, + }, + Orders: metadata.Orders{ + { + Name: "attributes.http.host", + }, + }, + }, + start: time.UnixMilli(1744880448784), + end: time.UnixMilli(1744884048785), + expected: "SELECT CAST(attributes[\"http.host\"] AS STRING) AS `attributes__bk_46__http__bk_46__host`, COUNT(CAST(attributes[\"http.host\"] AS STRING)) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bkapm_trace_bkop_doris`.doris WHERE `dtEventTimeStamp` >= 1744880448784 AND `dtEventTimeStamp` <= 1744884048785 AND `thedate` = '20250417' AND CAST(attributes[\"http.host\"] AS STRING) IS NOT NULL GROUP BY attributes__bk_46__http__bk_46__host, _timestamp_ ORDER BY CAST(attributes[\"http.host\"] AS STRING) DESC, `_timestamp_` ASC LIMIT 1", }, } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index acb34d16f..6bc79aaac 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -104,6 +104,7 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, newDim, isObject = d.dimTransform(dim) if isObject && d.encodeFunc != nil { selectAlias = fmt.Sprintf("%s AS `%s`", newDim, d.encodeFunc(dim)) + newDim = d.encodeFunc(dim) } else { selectAlias = newDim } @@ -483,7 +484,7 @@ func (d *DorisSQLExpr) dimTransform(s string) (string, bool) { fs := strings.Split(s, ".") if len(fs) > 1 { - return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `"]["`)), true + return fmt.Sprintf("CAST(%s[\"%s\"] AS STRING)", fs[0], strings.Join(fs[1:], `.`)), true } return fmt.Sprintf("`%s`", s), false } From 780c481fa6a0d94d231a2030d154849ecab44570 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 21 Apr 2025 17:01:15 +0800 Subject: [PATCH 094/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/query_ts.go | 7 +- .../query/structured/query_ts_test.go | 75 ++++++++ pkg/unify-query/service/http/query.go | 6 +- pkg/unify-query/service/http/query_test.go | 4 + pkg/unify-query/tsdb/bksql/format.go | 165 ++++++------------ pkg/unify-query/tsdb/bksql/instance_test.go | 70 +++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 25 ++- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 19 +- 8 files changed, 239 insertions(+), 132 deletions(-) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index afdef2fe7..2ed9325e8 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -150,10 +150,8 @@ func (q *QueryTs) ToQueryReference(ctx context.Context) (metadata.QueryReference } // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if query.HighLight != nil { - if !query.HighLight.Enable && q.HighLight.Enable { - query.HighLight = q.HighLight - } + if query.HighLight == nil && q.HighLight != nil { + query.HighLight = q.HighLight } // 复用字段配置，没有特殊配置的情况下使用公共配置 @@ -932,6 +930,7 @@ func (q *Query) BuildMetadataQuery( // 写入 ES 所需内容 query.QueryString = q.QueryString query.Source = q.KeepColumns + query.HighLight = q.HighLight query.Scroll = q.Scroll diff --git a/pkg/unify-query/query/structured/query_ts_test.go b/pkg/unify-query/query/structured/query_ts_test.go index 1975356bb..54ee67f66 100644 --- a/pkg/unify-query/query/structured/query_ts_test.go +++ b/pkg/unify-query/query/structured/query_ts_test.go @@ -1164,6 +1164,81 @@ func TestQueryTs_ToQueryReference(t *testing.T) { }, }, }, + "es 高亮查询": { + ts: &QueryTs{ + QueryList: []*Query{ + { + DataSource: BkLog, + TableID: "result_table.es", + FieldName: "usage", + ReferenceName: "a", + TimeAggregation: TimeAggregation{ + Function: "sum_over_time", + Window: "1m", + }, + AggregateMethodList: AggregateMethodList{ + { + Method: "sum", + Dimensions: []string{"__ext.container"}, + }, + { + Method: "topk", + VArgsList: []interface{}{ + 1, + }, + }, + }, + }, + }, + MetricMerge: "a", + Start: "1718865258", + End: "1718868858", + Step: "1m", + HighLight: &md.HighLight{ + Enable: true, + MaxAnalyzedOffset: 100, + }, + }, + isDirectQuery: false, + promql: `topk(1, sum by (__ext__bk_46__container) (last_over_time(a[1m])))`, + ref: md.QueryReference{ + "a": { + { + QueryList: md.QueryList{ + { + DataSource: BkLog, + Timezone: "UTC", + TableID: "result_table.es", + DataLabel: "es", + DB: "es_index", + MetricName: "usage", + VmConditionNum: 1, + VmCondition: `__name__="usage_value"`, + StorageID: "3", + StorageIDs: []string{ + "3", + }, + Field: "usage", + StorageType: consul.ElasticsearchStorageType, + Aggregates: md.Aggregates{ + { + Name: "sum", + Dimensions: []string{"__ext.container"}, + Window: time.Minute, + }, + }, + HighLight: &md.HighLight{ + Enable: true, + MaxAnalyzedOffset: 100, + }, + }, + }, + MetricName: "usage", + ReferenceName: "a", + }, + }, + }, + }, } { t.Run(name, func(t *testing.T) { var ( diff --git a/pkg/unify-query/service/http/query.go b/pkg/unify-query/service/http/query.go index 439a31002..7a1f5e88d 100644 --- a/pkg/unify-query/service/http/query.go +++ b/pkg/unify-query/service/http/query.go @@ -224,10 +224,8 @@ func queryRawWithInstance(ctx context.Context, queryTs *structured.QueryTs) (tot } // 复用 高亮配置，没有特殊配置的情况下使用公共配置 - if ql.HighLight != nil { - if !ql.HighLight.Enable && queryTs.HighLight.Enable { - ql.HighLight = queryTs.HighLight - } + if ql.HighLight == nil && queryTs.HighLight != nil { + ql.HighLight = queryTs.HighLight } // 复用字段配置，没有特殊配置的情况下使用公共配置 diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index 73ad7ceef..75eba5e60 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -840,6 +840,10 @@ func TestQueryRawWithInstance(t *testing.T) { start := "1723594000" end := "1723595000" + mock.BkSQL.Set(map[string]any{ + "SHOW CREATE TABLE `2_bklog_bkunify_query_doris`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":43,"bkbase_meta_api":0,"bkbase_apigw_api":33},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"double","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":0,"query_db":5,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":45,"match_query_routing_rule":0,"check_permission":43,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_bkunify_query_doris_2","total_record_size":11776,"timetaken":0.096,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + }) + mock.Es.Set(map[string]any{ `{"_source":{"includes":["__ext.container_id","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":301,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c726c895a380ba1a9df04ba4a977b29b","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"fa209967d4a8c5d21b3e4f67d2cd579e","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"dc888e9a3789976aa11483626fc61a4f","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c2dae031f095fa4b9deccf81964c7837","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8a916e558c71d4226f1d7f3279cf0fdd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"f6950fef394e813999d7316cdbf0de4d","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"328d487e284703b1d0bb8017dba46124","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"cb790ecb36bbaf02f6f0eb80ac2fd65c","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"bd8a8ef60e94ade63c55c8773170d458","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c8401bb4ec021b038cb374593b8adce3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594161000","__ext":{"container_id":"77bd897e66402eb66ee97a1f832fb55b2114d83dc369f01e36ce4cec8483786f"}}}]}}`, `{"_source":{"includes":["__ext.io_kubernetes_pod","dtEventTimeStamp"]},"from":0,"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1723594000,"include_lower":true,"include_upper":true,"to":1723595000}}}}},"size":20,"sort":[{"dtEventTimeStamp":{"order":"desc"}}]}`: `{"took":468,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":0.0,"hits":[{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"e058129ae18bff87c95e83f24584e654","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c124dae69af9b86a7128ee4281820158","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"c7f73abf7e865a4b4d7fc608387d01cf","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"39c3ec662881e44bf26d2a6bfc0e35c3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"58e03ce0b9754bf0657d49a5513adcb5","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"43a36f412886bf30b0746562513638d3","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"218ceafd04f89b39cda7954e51f4a48a","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"8d9abe9b782fe3a1272c93f0af6b39e1","_score":0.0,"_source":{"dtEventTimeStamp":"1723594211000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-599f9"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"0826407be7f04f19086774ed68eac8dd","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}},{"_index":"v2_2_bklog_bk_unify_query_20240814_0","_type":"_doc","_id":"d56b4120194eb37f53410780da777d43","_score":0.0,"_source":{"dtEventTimeStamp":"1723594224000","__ext":{"io_kubernetes_pod":"bkmonitor-unify-query-64bd4f5df4-llp94"}}}]}}`, diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 45ce8f311..9cd371751 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -60,7 +60,8 @@ type QueryFactory struct { start time.Time end time.Time - step time.Duration + + timeAggregate sqlExpr.TimeAggregate orders metadata.Orders @@ -157,7 +158,10 @@ func (f *QueryFactory) FormatDataToQueryResult(ctx context.Context, list []map[s encodeFunc := metadata.GetPromDataFormat(ctx).EncodeFunc() // 获取 metricLabel metricLabel := f.query.MetricLabels(ctx) + tsMap := map[string]*prompb.TimeSeries{} + tsTimeMap := make(map[string]map[int64]float64) + isAddZero := f.timeAggregate.Window > 0 && f.expr.Type() == sqlExpr.Doris // 先获取维度的 key 保证顺序一致 keys := make([]string, 0) @@ -260,16 +264,52 @@ func (f *QueryFactory) FormatDataToQueryResult(ctx context.Context, list []map[s } } - tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ - Value: vv, - Timestamp: vt, - }) + // 如果是时间聚合需要进行补零，否则直接返回 + if isAddZero { + if _, ok := tsTimeMap[key]; !ok { + tsTimeMap[key] = make(map[int64]float64) + } + + tsTimeMap[key][vt] = vv + } else { + tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ + Value: vv, + Timestamp: vt, + }) + } } // 转换结构体 res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) - for _, ts := range tsMap { - res.Timeseries = append(res.Timeseries, ts) + + // 如果是时间聚合需要进行补零，否则直接返回 + if isAddZero { + var ( + start time.Time + end time.Time + ) + + startMilli := (f.start.UnixMilli()+f.timeAggregate.OffsetMillis)/f.timeAggregate.Window.Milliseconds()*f.timeAggregate.Window.Milliseconds() - f.timeAggregate.OffsetMillis + start = time.UnixMilli(startMilli) + end = f.end + + for key, ts := range tsMap { + for i := start; end.Sub(i) > 0; i = i.Add(f.timeAggregate.Window) { + sample := prompb.Sample{ + Timestamp: i.UnixMilli(), + Value: 0, + } + if v, ok := tsTimeMap[key][i.UnixMilli()]; ok { + sample.Value = v + } + ts.Samples = append(ts.Samples, sample) + } + res.Timeseries = append(res.Timeseries, ts) + } + } else { + for _, ts := range tsMap { + res.Timeseries = append(res.Timeseries, ts) + } } return res, nil @@ -347,14 +387,17 @@ func (f *QueryFactory) SQL() (sql string, err error) { _, span = trace.NewSpan(f.ctx, "make-sql") defer span.End(&err) - selectFields, groupFields, orderFields, err := f.expr.ParserAggregatesAndOrders(f.query.Aggregates, f.orders) + selectFields, groupFields, orderFields, timeAggregate, err := f.expr.ParserAggregatesAndOrders(f.query.Aggregates, f.orders) if err != nil { return } + f.timeAggregate = timeAggregate + span.Set("select-fields", selectFields) span.Set("group-fields", groupFields) span.Set("order-fields", orderFields) + span.Set("timeAggregate", timeAggregate) sql += fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectFields, ", "), f.Table()) whereString, err := f.BuildWhere() @@ -402,109 +445,3 @@ func (f *QueryFactory) dims(dims []string, field string) []string { } return dimensions } - -func (f *QueryFactory) FormatData(keys []string, list []map[string]interface{}) (*prompb.QueryResult, error) { - res := &prompb.QueryResult{} - - if len(list) == 0 { - return res, nil - } - // 维度结构体为空则任务异常 - if len(keys) == 0 { - return res, fmt.Errorf("SelectFieldsOrder is empty") - } - - // 获取该指标的维度 key - field := f.query.Field - dimensions := f.dims(keys, field) - - tsMap := make(map[string]*prompb.TimeSeries, 0) - for _, d := range list { - // 优先获取时间和值 - var ( - vt int64 - vv float64 - - vtLong interface{} - vvDouble interface{} - - ok bool - ) - - if d == nil { - continue - } - - // 获取时间戳，单位是毫秒 - if vtLong, ok = d[sqlExpr.TimeStamp]; !ok { - vtLong = 0 - } - - if vtLong == nil { - continue - } - switch vtLong.(type) { - case int64: - vt = vtLong.(int64) - case float64: - vt = int64(vtLong.(float64)) - default: - return res, fmt.Errorf("%s type is error %T, %v", f.timeField, vtLong, vtLong) - } - - // 获取值 - if vvDouble, ok = d[field]; !ok { - return res, fmt.Errorf("dimension %s is emtpy", field) - } - - if vvDouble == nil { - continue - } - switch vvDouble.(type) { - case int64: - vv = float64(vvDouble.(int64)) - case float64: - vv = vvDouble.(float64) - default: - return res, fmt.Errorf("%s type is error %T, %v", field, vvDouble, vvDouble) - } - - var buf strings.Builder - lbl := make([]prompb.Label, 0, len(dimensions)) - // 获取维度信息 - for _, dimName := range dimensions { - val, err := getValue(dimName, d) - if err != nil { - return res, fmt.Errorf("dimensions %+v %s", dimensions, err.Error()) - } - - buf.WriteString(fmt.Sprintf("%s:%s,", dimName, val)) - lbl = append(lbl, prompb.Label{ - Name: dimName, - Value: val, - }) - } - - // 同一个 series 进行合并分组 - key := buf.String() - if _, ok := tsMap[key]; !ok { - tsMap[key] = &prompb.TimeSeries{ - Labels: lbl, - Samples: make([]prompb.Sample, 0), - } - } - - tsMap[key].Samples = append(tsMap[key].Samples, prompb.Sample{ - Value: vv, - Timestamp: vt, - }) - } - - // 转换结构体 - res.Timeseries = make([]*prompb.TimeSeries, 0, len(tsMap)) - for _, ts := range tsMap { - res.Timeseries = append(res.Timeseries, ts) - } - - return res, nil -} diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index af11f5350..619e57d1d 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -78,8 +78,13 @@ func TestInstance_QuerySeriesSet(t *testing.T) { "SELECT `cloudId`, COUNT(`cloudId`) AS `_value_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `cloudId` LIMIT 10005": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_bkunify_query_doris":{"start":"2025041100","end":"2025041123"}},"cluster":"doris-test","totalRecords":1,"external_api_call_time_mills":{"bkbase_auth_api":32,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"cloudId":0.0,"_value_":6}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":22,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":44,"match_query_routing_rule":0,"check_permission":32,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["cloudId","_value_"],"total_record_size":456,"timetaken":0.103,"result_schema":[{"field_type":"double","field_name":"__c0","field_alias":"cloudId","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"_value_","field_index":1}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_bkunify_query_doris"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, - "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + "SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY `namespace` LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":11,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"namespace\":\"bgp2\",\"_value_\":5},{\"namespace\":\"cq100\",\"_value_\":5},{\"namespace\":\"gz100\",\"_value_\":5},{\"namespace\":\"hn0-new\",\"_value_\":5},{\"namespace\":\"hn1\",\"_value_\":5},{\"namespace\":\"hn10\",\"_value_\":5},{\"namespace\":\"nj100\",\"_value_\":5},{\"namespace\":\"njloadtest\",\"_value_\":5},{\"namespace\":\"pbe\",\"_value_\":5},{\"namespace\":\"tj100\",\"_value_\":5},{\"namespace\":\"tj101\",\"_value_\":5}],\"select_fields_order\":[\"namespace\",\"_value_\"],\"sql\":\"SELECT `namespace`, COUNT(`login_rate`) AS `_value_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `namespace` LIMIT 10005\",\"total_record_size\":3216,\"timetaken\":0.24,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"5c70526f101a00531ef8fbaadc783693\",\"span_id\":\"2a31369ceb208970\"}", + + // count by 1m with mysql "SELECT COUNT(`login_rate`) AS `_value_`, MAX((dtEventTimeStamp + 0) / 60000 * 60000 - 0) AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY (dtEventTimeStamp + 0) / 60000 * 60000 - 0 ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":11,\"_timestamp_\":1730118600000},{\"_value_\":11,\"_timestamp_\":1730118660000},{\"_value_\":11,\"_timestamp_\":1730118720000},{\"_value_\":11,\"_timestamp_\":1730118780000},{\"_value_\":11,\"_timestamp_\":1730118840000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", + + // count by 1m with doris + "SELECT COUNT(`login_rate`) AS `_value_`, ((CAST((__shard_key__ / 1000 + 0) / 1 AS INT) * 1 - 0) * 60 * 1000) AS `_timestamp_` FROM `2_bklog_bkunify_query_doris`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' GROUP BY _timestamp_ ORDER BY `_timestamp_` ASC LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"_value_\":2,\"_timestamp_\":1730118720000}],\"select_fields_order\":[\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT COUNT(`login_rate`) AS `_value_`, MAX(`dtEventTimeStamp` - ((`dtEventTimeStamp` - 0) % 60000 - 0)) AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE (`dtEventTimeStamp` >= 1730118589181) AND (`dtEventTimeStamp` < 1730118889181) GROUP BY `dtEventTimeStamp` - (`dtEventTimeStamp` % 60000) ORDER BY `_timestamp_` LIMIT 10005\",\"total_record_size\":1424,\"timetaken\":0.231,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"127866cb51f85a4a7f620eb0e66588b1\",\"span_id\":\"578f26767bbb78c8\"}", }) end := time.UnixMilli(1730118889181) @@ -140,7 +145,68 @@ func TestInstance_QuerySeriesSet(t *testing.T) { }, }, }, - expected: `[{"labels":[{"name":"__name__","value":"bkdata:132_lol_new_login_queue_login_1min:default:login_rate"}],"samples":[{"value":11,"timestamp":1730118600000},{"value":11,"timestamp":1730118660000},{"value":11,"timestamp":1730118720000},{"value":11,"timestamp":1730118780000},{"value":11,"timestamp":1730118840000}],"exemplars":null,"histograms":null}]`, + expected: `[ { + "labels" : [ { + "name" : "__name__", + "value" : "bkdata:132_lol_new_login_queue_login_1min:default:login_rate" + } ], + "samples" : [{ + "value" : 11, + "timestamp" : 1730118600000 + }, { + "value" : 11, + "timestamp" : 1730118660000 + }, { + "value" : 11, + "timestamp" : 1730118720000 + }, { + "value" : 11, + "timestamp" : 1730118780000 + }, { + "value" : 11, + "timestamp" : 1730118840000 + } ], + "exemplars" : null, + "histograms" : null +} ]`, + }, + "count with 1m with doris": { + query: &metadata.Query{ + DataSource: datasource, + TableID: tableID, + DB: "2_bklog_bkunify_query_doris", + Measurement: sqlExpr.Doris, + MetricName: field, + DataLabel: db, + Aggregates: metadata.Aggregates{ + { + Name: "count", + Window: time.Minute, + }, + }, + }, + expected: `[ { + "labels" : [ { + "name" : "__name__", + "value" : "bkdata:132_lol_new_login_queue_login_1min:default:login_rate" + } ], + "samples" : [ { + "timestamp" : 1730118540000 + }, { + "timestamp" : 1730118600000 + }, + "timestamp" : 1730118660000 + }, { + "value" : 2, + "timestamp" : 1730118720000 + }, { + "timestamp" : 1730118780000 + }, { + "timestamp" : 1730118840000 + } ], + "exemplars" : null, + "histograms" : null +} ]`, }, } { t.Run(name, func(t *testing.T) { diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index c7366c8ea..09ff20a1a 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -34,6 +34,11 @@ var ( ErrorMatchAll = "不支持全字段检索" ) +type TimeAggregate struct { + Window time.Duration + OffsetMillis int64 +} + // SQLExpr 定义SQL表达式生成接口 // 接口包含字段映射、查询字符串解析、全条件解析等功能 type SQLExpr interface { @@ -50,11 +55,13 @@ type SQLExpr interface { // ParserAllConditions 解析全量条件生成SQL条件表达式 ParserAllConditions(allConditions metadata.AllConditions) (string, error) // ParserAggregatesAndOrders 解析聚合条件生成SQL条件表达式 - ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, error) + ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) ([]string, []string, []string, TimeAggregate, error) // DescribeTableSQL 返回当前表结构 DescribeTableSQL(table string) string // FieldMap 返回当前表结构 FieldMap() map[string]string + // Type 返回表达式类型 + Type() string } // SQL表达式注册管理相关变量 @@ -117,6 +124,10 @@ type DefaultSQLExpr struct { valueField string } +func (d *DefaultSQLExpr) Type() string { + return "default" +} + func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -148,7 +159,7 @@ func (d *DefaultSQLExpr) ParserQueryString(_ string) (string, error) { } // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 -func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { +func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, timeAggregate TimeAggregate, err error) { valueField, err := d.dimTransform(d.valueField) if err != nil { return @@ -156,6 +167,7 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate var ( window time.Duration + offsetMillis int64 timezone string dimensionMap = make(map[string]struct{}) ) @@ -191,7 +203,6 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate } if window > 0 { - var offsetMillis int if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { // 时间聚合函数兼容时区 loc, locErr := time.LoadLocation(timezone) @@ -200,7 +211,7 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate } // 获取时区偏移量 _, offset := time.Now().In(loc).Zone() - offsetMillis = offset * 1e3 + offsetMillis = int64(offset) * 1e3 } timeField := fmt.Sprintf("(%s + %d) / %d * %d - %d", d.timeField, offsetMillis, window.Milliseconds(), window.Milliseconds(), offsetMillis) @@ -250,6 +261,12 @@ func (d *DefaultSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregate orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) } + // 回传时间聚合信息 + timeAggregate = TimeAggregate{ + Window: window, + OffsetMillis: offsetMillis, + } + return } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 6bc79aaac..863b79a7d 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -39,6 +39,10 @@ type DorisSQLExpr struct { var _ SQLExpr = (*DorisSQLExpr)(nil) +func (d *DorisSQLExpr) Type() string { + return Doris +} + func (d *DorisSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -81,11 +85,13 @@ func (d *DorisSQLExpr) DescribeTableSQL(table string) string { } // ParserAggregatesAndOrders 解析聚合函数，生成 select 和 group by 字段 -func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, err error) { +func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orders metadata.Orders) (selectFields []string, groupByFields []string, orderByFields []string, timeAggregate TimeAggregate, err error) { valueField, _ := d.dimTransform(d.valueField) var ( - window time.Duration + window time.Duration + offsetMinutes int64 + timezone string dimensionMap = map[string]struct{}{ FieldValue: {}, @@ -136,7 +142,6 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, if window > 0 { // 获取时区偏移量 - var offsetMinutes int // 如果是按天聚合，则增加时区偏移量 if window.Milliseconds()%(24*time.Hour).Milliseconds() == 0 { // 时间聚合函数兼容时区 @@ -145,7 +150,7 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, loc = time.UTC } _, offset := time.Now().In(loc).Zone() - offsetMinutes = offset / 60 + offsetMinutes = int64(offset) / 60 } // 如果是按照分钟聚合，则使用 __shard_key__ 作为时间字段 @@ -204,6 +209,12 @@ func (d *DorisSQLExpr) ParserAggregatesAndOrders(aggregates metadata.Aggregates, orderByFields = append(orderByFields, fmt.Sprintf("%s %s", orderField, ascName)) } + // 回传时间聚合信息 + timeAggregate = TimeAggregate{ + Window: window, + OffsetMillis: offsetMinutes, + } + return } From fc219a5e5fd7ec6eb8f94ef38ef31a8dba90cf63 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Mon, 21 Apr 2025 17:11:09 +0800 Subject: [PATCH 095/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/instance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 619e57d1d..9ff183d27 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -194,7 +194,7 @@ func TestInstance_QuerySeriesSet(t *testing.T) { "timestamp" : 1730118540000 }, { "timestamp" : 1730118600000 - }, + }, { "timestamp" : 1730118660000 }, { "value" : 2, From 16c57cee52e80cb6ffebf264128d1720c1c706bf Mon Sep 17 00:00:00 2001 From: shamcleren Date: Tue, 22 Apr 2025 15:15:12 +0800 Subject: [PATCH 096/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/query/structured/query_ts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 2ed9325e8..10c7cfc94 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -893,7 +893,7 @@ func (q *Query) BuildMetadataQuery( // 兼容原逻辑，storageType 通过 storageMap 获取 if query.StorageType == "" { - log.Warnf(ctx, "storageType is empty with %s", query.TableID) + log.Warnf(ctx, "storageType is empty with %s", tsDB.TableID) stg, _ := tsdb.GetStorage(query.StorageID) if stg != nil { query.StorageType = stg.Type From 53590fa7c632154e33d28d90090c37b1b25c5ed0 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 23 Apr 2025 12:04:38 +0800 Subject: [PATCH 097/104] =?UTF-8?q?feat:=20=E9=9D=9E=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E8=81=9A=E5=90=88=E6=8E=A5=E5=8F=A3=E6=8E=92=E5=BA=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E4=BF=AE=E6=94=B9=20#1010158081123667928?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../function}/merge_series.go | 8 +- .../function}/merge_series_test.go | 12 +- pkg/unify-query/internal/function/samples.go | 6 + pkg/unify-query/service/http/query_test.go | 10 +- pkg/unify-query/tsdb/elasticsearch/format.go | 14 +- .../tsdb/elasticsearch/format_test.go | 2 +- .../tsdb/elasticsearch/instance.go | 139 +++--------------- pkg/unify-query/tsdb/prometheus/querier.go | 2 +- 8 files changed, 55 insertions(+), 138 deletions(-) rename pkg/unify-query/{tsdb/prometheus => internal/function}/merge_series.go (95%) rename pkg/unify-query/{tsdb/prometheus => internal/function}/merge_series_test.go (96%) diff --git a/pkg/unify-query/tsdb/prometheus/merge_series.go b/pkg/unify-query/internal/function/merge_series.go similarity index 95% rename from pkg/unify-query/tsdb/prometheus/merge_series.go rename to pkg/unify-query/internal/function/merge_series.go index 7229d22b0..f3fe2b1d2 100644 --- a/pkg/unify-query/tsdb/prometheus/merge_series.go +++ b/pkg/unify-query/internal/function/merge_series.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package prometheus +package function import ( "sort" @@ -17,8 +17,6 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" - - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/query/structured" ) func NewMergeSeriesSetWithFuncAndSort(name string) func(...storage.Series) storage.Series { @@ -36,14 +34,14 @@ func NewMergeSeriesSetWithFuncAndSort(name string) func(...storage.Series) stora // 根据name选择聚合函数 var aggFunc func(float64, float64) float64 switch strings.ToLower(name) { - case structured.MIN, structured.MinOT: + case Min, MinOT: aggFunc = func(a, b float64) float64 { if a < b { return a } return b } - case structured.MAX, structured.MaxOT: + case Max, MaxOT: aggFunc = func(a, b float64) float64 { if a > b { return a diff --git a/pkg/unify-query/tsdb/prometheus/merge_series_test.go b/pkg/unify-query/internal/function/merge_series_test.go similarity index 96% rename from pkg/unify-query/tsdb/prometheus/merge_series_test.go rename to pkg/unify-query/internal/function/merge_series_test.go index 0e7a0d006..3240a6f2b 100644 --- a/pkg/unify-query/tsdb/prometheus/merge_series_test.go +++ b/pkg/unify-query/internal/function/merge_series_test.go @@ -7,7 +7,7 @@ // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -package prometheus_test +package function_test import ( "testing" @@ -17,8 +17,8 @@ import ( "github.com/prometheus/prometheus/storage/remote" "github.com/stretchr/testify/assert" + "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/function" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/mock" - "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/tsdb/prometheus" ) func TestMergeSeriesSet(t *testing.T) { @@ -158,7 +158,7 @@ func TestMergeSeriesSet(t *testing.T) { ts: mock.TimeSeriesList{ *ts2, *ts4, *ts1, *ts3, }, - fn: prometheus.NewMergeSeriesSetWithFuncAndSort(""), + fn: function.NewMergeSeriesSetWithFuncAndSort(""), }, "two queryResult with chainedSeriesMerge": { qrs: []*prompb.QueryResult{ @@ -291,7 +291,7 @@ func TestMergeSeriesSet(t *testing.T) { }, }, }, - fn: prometheus.NewMergeSeriesSetWithFuncAndSort(""), + fn: function.NewMergeSeriesSetWithFuncAndSort(""), }, "two queryResult with mergeSeriesSetWithFuncAndSort max": { qrs: []*prompb.QueryResult{ @@ -358,7 +358,7 @@ func TestMergeSeriesSet(t *testing.T) { }, }, }, - fn: prometheus.NewMergeSeriesSetWithFuncAndSort("max"), + fn: function.NewMergeSeriesSetWithFuncAndSort("max"), }, "two queryResult with mergeSeriesSetWithFuncAndSort min": { qrs: []*prompb.QueryResult{ @@ -425,7 +425,7 @@ func TestMergeSeriesSet(t *testing.T) { }, }, }, - fn: prometheus.NewMergeSeriesSetWithFuncAndSort("min"), + fn: function.NewMergeSeriesSetWithFuncAndSort("min"), }, } diff --git a/pkg/unify-query/internal/function/samples.go b/pkg/unify-query/internal/function/samples.go index c98114f88..2abc5e649 100644 --- a/pkg/unify-query/internal/function/samples.go +++ b/pkg/unify-query/internal/function/samples.go @@ -24,6 +24,12 @@ const ( Avg = "avg" Sum = "sum" Count = "count" + + MinOT = "min_over_time" + MaxOT = "max_over_time" + AvgOT = "avg_over_time" + SumOT = "sum_over_time" + CountOT = "count_over_time" ) // MergeSamplesWithFuncAndSort 合并 samples 数据，如果相同时间的进行函数处理，并且按照时间排序 diff --git a/pkg/unify-query/service/http/query_test.go b/pkg/unify-query/service/http/query_test.go index f83ca3903..3494408e8 100644 --- a/pkg/unify-query/service/http/query_test.go +++ b/pkg/unify-query/service/http/query_test.go @@ -148,9 +148,11 @@ func TestQueryReference(t *testing.T) { defaultEnd := time.UnixMilli(1741155879987) // 2025-03-05 14:24:39 mock.Es.Set(map[string]any{ - `{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1741154079123,"include_lower":true,"include_upper":true,"to":1741155879987}}}}},"size":0}`: `{"took":626,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182355}}}`, - `{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1741154079,"include_lower":true,"include_upper":true,"to":1741155879}}}}},"size":0}`: `{"took":171,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182486}}}`, - `{"aggregations":{"__ext.container_name":{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"terms":{"field":"__ext.container_name","missing":" ","order":[{"_value":"asc"}],"size":5}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1741154079123,"include_lower":true,"include_upper":true,"to":1741155879987}}}}},"size":0}`: `{"took":860,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"__ext.container_name":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"unify-query","doc_count":182355,"_value":{"value":182355}},{"key":" ","doc_count":182355,"_value":{"value":4325521}}]}}}`, + `{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1741154079123,"include_lower":true,"include_upper":true,"to":1741155879987}}}}},"size":0}`: `{"took":626,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182355}}}`, + `{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1741154079,"include_lower":true,"include_upper":true,"to":1741155879}}}}},"size":0}`: `{"took":171,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182486}}}`, + + `{"aggregations":{"__ext.container_name":{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"terms":{"field":"__ext.container_name","missing":" ","order":[{"_value":"asc"}],"size":5}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_millis","from":1741154079123,"include_lower":true,"include_upper":true,"to":1741155879987}}}}},"size":0}`: `{"took":860,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"__ext.container_name":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"unify-query","doc_count":182355,"_value":{"value":182355}},{"key":" ","doc_count":182355,"_value":{"value":4325521}}]}}}`, + `{"aggregations":{"__ext.container_name":{"aggregations":{"_value":{"value_count":{"field":"gseIndex"}}},"terms":{"field":"__ext.container_name","missing":" ","order":[{"_value":"desc"}],"size":5}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1741154079,"include_lower":true,"include_upper":true,"to":1741155879}}}}},"size":0}`: `{"took":885,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"__ext.container_name":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"unify-query","doc_count":182486,"_value":{"value":182486}}]}}}`, `{"aggregations":{"_value":{"value_count":{"field":"__ext.container_name"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1741154079,"include_lower":true,"include_upper":true,"to":1741155879}}}}},"size":0}`: `{"took":283,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182486}}}`, `{"aggregations":{"_value":{"value_count":{"field":"__ext.io_kubernetes_pod"}}},"query":{"bool":{"filter":{"range":{"dtEventTimeStamp":{"format":"epoch_second","from":1741154079,"include_lower":true,"include_upper":true,"to":1741155879}}}}},"size":0}`: `{"took":167,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"_value":{"value":182486}}}`, @@ -243,7 +245,7 @@ func TestQueryReference(t *testing.T) { Instant: true, SpaceUid: spaceUid, }, - result: `[{"name":"_result0","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["__ext.container_name"],"group_values":[""],"values":[[1741154079123,4325521]]},{"name":"_result1","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["__ext.container_name"],"group_values":["unify-query"],"values":[[1741154079123,182355]]}]`, + result: `[{"name":"_result0","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["__ext.container_name"],"group_values":["unify-query"],"values":[[1741154079123,182355]]},{"name":"_result1","metric_name":"","columns":["_time","_value"],"types":["float","float"],"group_keys":["__ext.container_name"],"group_values":[""],"values":[[1741154079123,4325521]]}]`, }, "根据维度 __ext.container_name 进行 count 聚合，同时用值倒序": { queryTs: &structured.QueryTs{ diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 2501c9289..5ec296c1d 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -397,7 +397,7 @@ func (f *FormatFactory) nestedAgg(key string) { } // AggDataFormat 解析 es 的聚合计算 -func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *prompb.Label) (map[string]*prompb.TimeSeries, error) { +func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *prompb.Label) (*prompb.QueryResult, error) { if data == nil { return nil, nil } @@ -424,6 +424,8 @@ func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *pr } timeSeriesMap := make(map[string]*prompb.TimeSeries) + keySort := make([]string, 0) + for _, im := range af.items { var ( tsLabels []prompb.Label @@ -448,6 +450,7 @@ func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *pr seriesKey := seriesNameBuilder.String() if _, ok := timeSeriesMap[seriesKey]; !ok { + keySort = append(keySort, seriesKey) timeSeriesMap[seriesKey] = &prompb.TimeSeries{ Samples: make([]prompb.Sample, 0), } @@ -464,7 +467,14 @@ func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *pr }) } - return timeSeriesMap, nil + tss := make([]*prompb.TimeSeries, 0, len(timeSeriesMap)) + for _, key := range keySort { + if ts, ok := timeSeriesMap[key]; ok { + tss = append(tss, ts) + } + } + + return &prompb.QueryResult{Timeseries: tss}, nil } func (f *FormatFactory) SetData(data map[string]any) { diff --git a/pkg/unify-query/tsdb/elasticsearch/format_test.go b/pkg/unify-query/tsdb/elasticsearch/format_test.go index 9862f9ff8..5d19b94c6 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/format_test.go @@ -419,7 +419,7 @@ func TestFormatFactory_AggDataFormat(t *testing.T) { }, }, res: `{"took":12,"timed_out":false,"_shards":{"total":9,"successful":9,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"database_name":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"dbRaphael","doc_count":31681,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":31681,"_value":{"value":6.49154281472E11}}]}},{"key":"bak_cbs_dbRaphael_new_20220117170335_100630794","doc_count":7599,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":7599,"_value":{"value":9.17192704E8}}]}},{"key":"dbRaphael_new","doc_count":7599,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":7599,"_value":{"value":9.17192704E8}}]}},{"key":"time_controller_prod","doc_count":1834,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1834,"_value":{"value":1.905320722432E12}}]}},{"key":"db_time","doc_count":1288,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1288,"_value":{"value":1.58793728E8}}]}},{"key":"db_gamerapp","doc_count":815,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":815,"_value":{"value":1.65445632E8}}]}},{"key":"analysis","doc_count":782,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":782,"_value":{"value":1.09520945152E11}}]}},{"key":"scrm","doc_count":753,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":251,"_value":{"value":2.48476893184E11}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":251,"_value":{"value":2.48476893184E11}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":251,"_value":{"value":2.48476893184E11}}]}},{"key":"dbRaphaelBiz","doc_count":222,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":222,"_value":{"value":6.4331317248E10}}]}},{"key":"boss","doc_count":130,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":130,"_value":{"value":9.814016E7}}]}},{"key":"db_gamematrix_web_manage_prod","doc_count":122,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":122,"_value":{"value":3.144843264E9}}]}},{"key":"opcg_backend_dev","doc_count":108,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":108,"_value":{"value":2.752512E7}}]}},{"key":"gmve_configcenter","doc_count":107,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":107,"_value":{"value":1.5253504E8}}]}},{"key":"opcg_backend_pro","doc_count":106,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":106,"_value":{"value":3.726671872E9}}]}},{"key":"db_vdesk_task","doc_count":99,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":33,"_value":{"value":1.99426048E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":33,"_value":{"value":1.99426048E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":33,"_value":{"value":1.99426048E8}}]}},{"key":"db_vdesk_config","doc_count":87,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":29,"_value":{"value":6.20756992E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":29,"_value":{"value":6.20756992E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":29,"_value":{"value":6.20756992E8}}]}},{"key":"db_vdesk_ticket","doc_count":87,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":29,"_value":{"value":5.23051008E9}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":29,"_value":{"value":5.238898688E9}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":29,"_value":{"value":5.2514816E9}}]}},{"key":"db_mc","doc_count":86,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":86,"_value":{"value":1.30160410624E11}}]}},{"key":"grafana","doc_count":77,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":77,"_value":{"value":6.1587456E7}}]}},{"key":"db_vdesk_customer","doc_count":57,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":19,"_value":{"value":1.0132996096E10}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":19,"_value":{"value":1.0132996096E10}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":19,"_value":{"value":1.0132996096E10}}]}},{"key":"db_xinyue_robot","doc_count":57,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":19,"_value":{"value":3.490217984E9}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":19,"_value":{"value":3.490217984E9}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":19,"_value":{"value":3.490217984E9}}]}},{"key":"db_cloudgames","doc_count":38,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":38,"_value":{"value":1.7596563456E10}}]}},{"key":"db_scpal","doc_count":33,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":11,"_value":{"value":1.0747904E7}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":11,"_value":{"value":1.0747904E7}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":11,"_value":{"value":1.0747904E7}}]}},{"key":"db_activity","doc_count":30,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":10,"_value":{"value":2097152.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":10,"_value":{"value":2097152.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":2097152.0}}]}},{"key":"db_vdesk_im","doc_count":30,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":10,"_value":{"value":8.1846272E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":10,"_value":{"value":8.3091456E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":8.3091456E8}}]}},{"key":"opcg_cluster","doc_count":27,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":27,"_value":{"value":2.0987904E7}}]}},{"key":"opcg_cluster_test","doc_count":27,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":27,"_value":{"value":3.3882112E7}}]}},{"key":"offline","doc_count":26,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":26,"_value":{"value":7.0555746304E10}}]}},{"key":"db_backup","doc_count":24,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":8,"_value":{"value":2.0854243328E10}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":8,"_value":{"value":2.0359315456E10}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":8,"_value":{"value":2.0455784448E10}}]}},{"key":"db_live_act","doc_count":21,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":7,"_value":{"value":1343488.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":7,"_value":{"value":1343488.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":7,"_value":{"value":1343488.0}}]}},{"key":"db_svip_data_flow","doc_count":21,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":7,"_value":{"value":4.20724736E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":7,"_value":{"value":4.20724736E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":7,"_value":{"value":4.20724736E8}}]}},{"key":"db_h5_backend","doc_count":18,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":18,"_value":{"value":1.45408E8}}]}},{"key":"sr_server","doc_count":17,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":17,"_value":{"value":1.7973248E7}}]}},{"key":"db_mp","doc_count":16,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":16,"_value":{"value":1064960.0}}]}},{"key":"games_launcher","doc_count":16,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":16,"_value":{"value":3.1309824E7}}]}},{"key":"db_external_idata","doc_count":15,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":15,"_value":{"value":4.674715648E9}}]}},{"key":"db_robot_platform","doc_count":15,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":5,"_value":{"value":1.18902964224E11}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":5,"_value":{"value":1.18902964224E11}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":5,"_value":{"value":1.18902964224E11}}]}},{"key":"db_self_service","doc_count":15,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":5,"_value":{"value":5.2723712E7}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":5,"_value":{"value":5.2723712E7}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":5,"_value":{"value":5.2723712E7}}]}},{"key":"test_tv_backend_db","doc_count":13,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":13,"_value":{"value":1327104.0}}]}},{"key":"db_ocpauth","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":1589248.0}}]}},{"key":"gmve_configcenter_cq4","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":8.331264E7}}]}},{"key":"gmve_configcenter_cq5","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":9.1766784E7}}]}},{"key":"gmve_configcenter_nj6","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":8.3378176E7}}]}},{"key":"gmve_configcenter_sz3","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":1.08412928E8}}]}},{"key":"gmve_configcenter_tj4","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":4.8627712E7}}]}},{"key":"gmve_configcenter_tj7","doc_count":12,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":12,"_value":{"value":7.90528E7}}]}},{"key":"deploy_gmve_configcenter","doc_count":11,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":11,"_value":{"value":1556480.0}}]}},{"key":"opcg_db_hub","doc_count":11,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":11,"_value":{"value":2.855911424E9}}]}},{"key":"paladin","doc_count":11,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":11,"_value":{"value":1.1010048E7}}]}},{"key":"tmp_gmve_configcenter","doc_count":11,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":11,"_value":{"value":1.0944512E7}}]}},{"key":"cloudgame","doc_count":10,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":8.3951616E7}}]}},{"key":"dev_gmve_configcenter","doc_count":10,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":1.2713984E7}}]}},{"key":"qa1_gmve_configcenter","doc_count":10,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":4.3220992E7}}]}},{"key":"rc_gmve_configcenter","doc_count":10,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":10,"_value":{"value":2.5280512E7}}]}},{"key":"dbXyplusService","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":1.1010048E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":1.1010048E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":1.1010048E8}}]}},{"key":"db_live_vote","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":458752.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":458752.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":458752.0}}]}},{"key":"db_team","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":950272.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":950272.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":950272.0}}]}},{"key":"db_vdesk_approval","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":1.1845632E7}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":1.1845632E7}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":1.1845632E7}}]}},{"key":"db_vdesk_satisfaction","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":376832.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":376832.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":376832.0}}]}},{"key":"db_vdesk_voice","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":1343488.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":1343488.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":1343488.0}}]}},{"key":"db_xinyue_robot_act","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":2.0178796544E10}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":2.0178796544E10}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":2.0178796544E10}}]}},{"key":"qa2_gmve_configcenter","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":9,"_value":{"value":1.04644608E8}}]}},{"key":"test","doc_count":9,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":3,"_value":{"value":3.2702464E7}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":3,"_value":{"value":3.2702464E7}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":3.2702464E7}}]}},{"key":"sr_server_qa2","doc_count":8,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":8,"_value":{"value":3.0343168E7}}]}},{"key":"zhiqiangli_test","doc_count":8,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":8,"_value":{"value":819200.0}}]}},{"key":"sr_server_rc2","doc_count":7,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":7,"_value":{"value":1163264.0}}]}},{"key":"db_danmu","doc_count":6,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":6,"_value":{"value":1835008.0}}]}},{"key":"db_svip_station","doc_count":6,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":2,"_value":{"value":278528.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":2,"_value":{"value":278528.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":278528.0}}]}},{"key":"gmve_license_center","doc_count":6,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":6,"_value":{"value":344064.0}}]}},{"key":"data","doc_count":5,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":5,"_value":{"value":4.6137344E7}}]}},{"key":"db_inner_idata","doc_count":5,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":5,"_value":{"value":3.2751616E7}}]}},{"key":"opcgvirt_gmatrix","doc_count":5,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":5,"_value":{"value":1.70098688E8}}]}},{"key":"digitalgw","doc_count":4,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":4,"_value":{"value":1.333788672E9}}]}},{"key":"db_game","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":1,"_value":{"value":98304.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":1,"_value":{"value":98304.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":98304.0}}]}},{"key":"db_hostPasswd","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":7.7709312E7}}]}},{"key":"db_robot_oper_closedloop","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":1,"_value":{"value":1.04562688E8}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":1,"_value":{"value":1.04562688E8}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":1.04562688E8}}]}},{"key":"db_user","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":1,"_value":{"value":163840.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":1,"_value":{"value":163840.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":163840.0}}]}},{"key":"db_vdesk_llm","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":1,"_value":{"value":98304.0}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":1,"_value":{"value":98304.0}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":98304.0}}]}},{"key":"db_xinyue_robot_log","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":1,"_value":{"value":2.0356399104E10}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":1,"_value":{"value":2.0356399104E10}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":2.0356399104E10}}]}},{"key":"dbtest","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":114688.0}}]}},{"key":"games@002dlauncher@002danalysis","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":311296.0}}]}},{"key":"gmve_license","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":294912.0}}]}},{"key":"greatwall","doc_count":3,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":3,"_value":{"value":1310720.0}}]}},{"key":"codeaxis","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":9584640.0}}]}},{"key":"dev_digitalgw","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":507904.0}}]}},{"key":"dev_dunhuang","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":294912.0}}]}},{"key":"dev_gmve_recorder","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":458752.0}}]}},{"key":"dunhuang","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":4.483710976E9}}]}},{"key":"sr_server_test","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":5652480.0}}]}},{"key":"tdw_export","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":311296.0}}]}},{"key":"test_gmve_recorder","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":557056.0}}]}},{"key":"tiyan_gmve_recorder","doc_count":2,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":2,"_value":{"value":229376.0}}]}},{"key":"bak_cbs_cloudgame_201911131726","doc_count":1,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":147456.0}}]}},{"key":"test_db","doc_count":1,"dtEventTimeStamp":{"buckets":[{"key_as_string":"1733068800000","key":1733068800000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733155200000","key":1733155200000,"doc_count":0,"_value":{"value":null}},{"key_as_string":"1733241600000","key":1733241600000,"doc_count":1,"_value":{"value":114688.0}}]}}]}}}`, - expected: `{"name:\"database_name\" value:\"analysis\" ":{"labels":[{"name":"database_name","value":"analysis"}],"samples":[{"value":109520945152,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"bak_cbs_cloudgame_201911131726\" ":{"labels":[{"name":"database_name","value":"bak_cbs_cloudgame_201911131726"}],"samples":[{"value":147456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"bak_cbs_dbRaphael_new_20220117170335_100630794\" ":{"labels":[{"name":"database_name","value":"bak_cbs_dbRaphael_new_20220117170335_100630794"}],"samples":[{"value":917192704,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"boss\" ":{"labels":[{"name":"database_name","value":"boss"}],"samples":[{"value":98140160,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"cloudgame\" ":{"labels":[{"name":"database_name","value":"cloudgame"}],"samples":[{"value":83951616,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"codeaxis\" ":{"labels":[{"name":"database_name","value":"codeaxis"}],"samples":[{"value":9584640,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"data\" ":{"labels":[{"name":"database_name","value":"data"}],"samples":[{"value":46137344,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dbRaphael\" ":{"labels":[{"name":"database_name","value":"dbRaphael"}],"samples":[{"value":649154281472,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dbRaphaelBiz\" ":{"labels":[{"name":"database_name","value":"dbRaphaelBiz"}],"samples":[{"value":64331317248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dbRaphael_new\" ":{"labels":[{"name":"database_name","value":"dbRaphael_new"}],"samples":[{"value":917192704,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dbXyplusService\" ":{"labels":[{"name":"database_name","value":"dbXyplusService"}],"samples":[{"value":110100480,"timestamp":1733068800000},{"value":110100480,"timestamp":1733155200000},{"value":110100480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_activity\" ":{"labels":[{"name":"database_name","value":"db_activity"}],"samples":[{"value":2097152,"timestamp":1733068800000},{"value":2097152,"timestamp":1733155200000},{"value":2097152,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_backup\" ":{"labels":[{"name":"database_name","value":"db_backup"}],"samples":[{"value":20854243328,"timestamp":1733068800000},{"value":20359315456,"timestamp":1733155200000},{"value":20455784448,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_cloudgames\" ":{"labels":[{"name":"database_name","value":"db_cloudgames"}],"samples":[{"value":17596563456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_danmu\" ":{"labels":[{"name":"database_name","value":"db_danmu"}],"samples":[{"value":1835008,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_external_idata\" ":{"labels":[{"name":"database_name","value":"db_external_idata"}],"samples":[{"value":4674715648,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_game\" ":{"labels":[{"name":"database_name","value":"db_game"}],"samples":[{"value":98304,"timestamp":1733068800000},{"value":98304,"timestamp":1733155200000},{"value":98304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_gamematrix_web_manage_prod\" ":{"labels":[{"name":"database_name","value":"db_gamematrix_web_manage_prod"}],"samples":[{"value":3144843264,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_gamerapp\" ":{"labels":[{"name":"database_name","value":"db_gamerapp"}],"samples":[{"value":165445632,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_h5_backend\" ":{"labels":[{"name":"database_name","value":"db_h5_backend"}],"samples":[{"value":145408000,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_hostPasswd\" ":{"labels":[{"name":"database_name","value":"db_hostPasswd"}],"samples":[{"value":77709312,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_inner_idata\" ":{"labels":[{"name":"database_name","value":"db_inner_idata"}],"samples":[{"value":32751616,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_live_act\" ":{"labels":[{"name":"database_name","value":"db_live_act"}],"samples":[{"value":1343488,"timestamp":1733068800000},{"value":1343488,"timestamp":1733155200000},{"value":1343488,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_live_vote\" ":{"labels":[{"name":"database_name","value":"db_live_vote"}],"samples":[{"value":458752,"timestamp":1733068800000},{"value":458752,"timestamp":1733155200000},{"value":458752,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_mc\" ":{"labels":[{"name":"database_name","value":"db_mc"}],"samples":[{"value":130160410624,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_mp\" ":{"labels":[{"name":"database_name","value":"db_mp"}],"samples":[{"value":1064960,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_ocpauth\" ":{"labels":[{"name":"database_name","value":"db_ocpauth"}],"samples":[{"value":1589248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_robot_oper_closedloop\" ":{"labels":[{"name":"database_name","value":"db_robot_oper_closedloop"}],"samples":[{"value":104562688,"timestamp":1733068800000},{"value":104562688,"timestamp":1733155200000},{"value":104562688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_robot_platform\" ":{"labels":[{"name":"database_name","value":"db_robot_platform"}],"samples":[{"value":118902964224,"timestamp":1733068800000},{"value":118902964224,"timestamp":1733155200000},{"value":118902964224,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_scpal\" ":{"labels":[{"name":"database_name","value":"db_scpal"}],"samples":[{"value":10747904,"timestamp":1733068800000},{"value":10747904,"timestamp":1733155200000},{"value":10747904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_self_service\" ":{"labels":[{"name":"database_name","value":"db_self_service"}],"samples":[{"value":52723712,"timestamp":1733068800000},{"value":52723712,"timestamp":1733155200000},{"value":52723712,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_svip_data_flow\" ":{"labels":[{"name":"database_name","value":"db_svip_data_flow"}],"samples":[{"value":420724736,"timestamp":1733068800000},{"value":420724736,"timestamp":1733155200000},{"value":420724736,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_svip_station\" ":{"labels":[{"name":"database_name","value":"db_svip_station"}],"samples":[{"value":278528,"timestamp":1733068800000},{"value":278528,"timestamp":1733155200000},{"value":278528,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_team\" ":{"labels":[{"name":"database_name","value":"db_team"}],"samples":[{"value":950272,"timestamp":1733068800000},{"value":950272,"timestamp":1733155200000},{"value":950272,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_time\" ":{"labels":[{"name":"database_name","value":"db_time"}],"samples":[{"value":158793728,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_user\" ":{"labels":[{"name":"database_name","value":"db_user"}],"samples":[{"value":163840,"timestamp":1733068800000},{"value":163840,"timestamp":1733155200000},{"value":163840,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_approval\" ":{"labels":[{"name":"database_name","value":"db_vdesk_approval"}],"samples":[{"value":11845632,"timestamp":1733068800000},{"value":11845632,"timestamp":1733155200000},{"value":11845632,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_config\" ":{"labels":[{"name":"database_name","value":"db_vdesk_config"}],"samples":[{"value":620756992,"timestamp":1733068800000},{"value":620756992,"timestamp":1733155200000},{"value":620756992,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_customer\" ":{"labels":[{"name":"database_name","value":"db_vdesk_customer"}],"samples":[{"value":10132996096,"timestamp":1733068800000},{"value":10132996096,"timestamp":1733155200000},{"value":10132996096,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_im\" ":{"labels":[{"name":"database_name","value":"db_vdesk_im"}],"samples":[{"value":818462720,"timestamp":1733068800000},{"value":830914560,"timestamp":1733155200000},{"value":830914560,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_llm\" ":{"labels":[{"name":"database_name","value":"db_vdesk_llm"}],"samples":[{"value":98304,"timestamp":1733068800000},{"value":98304,"timestamp":1733155200000},{"value":98304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_satisfaction\" ":{"labels":[{"name":"database_name","value":"db_vdesk_satisfaction"}],"samples":[{"value":376832,"timestamp":1733068800000},{"value":376832,"timestamp":1733155200000},{"value":376832,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_task\" ":{"labels":[{"name":"database_name","value":"db_vdesk_task"}],"samples":[{"value":199426048,"timestamp":1733068800000},{"value":199426048,"timestamp":1733155200000},{"value":199426048,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_ticket\" ":{"labels":[{"name":"database_name","value":"db_vdesk_ticket"}],"samples":[{"value":5230510080,"timestamp":1733068800000},{"value":5238898688,"timestamp":1733155200000},{"value":5251481600,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_vdesk_voice\" ":{"labels":[{"name":"database_name","value":"db_vdesk_voice"}],"samples":[{"value":1343488,"timestamp":1733068800000},{"value":1343488,"timestamp":1733155200000},{"value":1343488,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_xinyue_robot\" ":{"labels":[{"name":"database_name","value":"db_xinyue_robot"}],"samples":[{"value":3490217984,"timestamp":1733068800000},{"value":3490217984,"timestamp":1733155200000},{"value":3490217984,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_xinyue_robot_act\" ":{"labels":[{"name":"database_name","value":"db_xinyue_robot_act"}],"samples":[{"value":20178796544,"timestamp":1733068800000},{"value":20178796544,"timestamp":1733155200000},{"value":20178796544,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"db_xinyue_robot_log\" ":{"labels":[{"name":"database_name","value":"db_xinyue_robot_log"}],"samples":[{"value":20356399104,"timestamp":1733068800000},{"value":20356399104,"timestamp":1733155200000},{"value":20356399104,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dbtest\" ":{"labels":[{"name":"database_name","value":"dbtest"}],"samples":[{"value":114688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"deploy_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"deploy_gmve_configcenter"}],"samples":[{"value":1556480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dev_digitalgw\" ":{"labels":[{"name":"database_name","value":"dev_digitalgw"}],"samples":[{"value":507904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dev_dunhuang\" ":{"labels":[{"name":"database_name","value":"dev_dunhuang"}],"samples":[{"value":294912,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dev_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"dev_gmve_configcenter"}],"samples":[{"value":12713984,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dev_gmve_recorder\" ":{"labels":[{"name":"database_name","value":"dev_gmve_recorder"}],"samples":[{"value":458752,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"digitalgw\" ":{"labels":[{"name":"database_name","value":"digitalgw"}],"samples":[{"value":1333788672,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"dunhuang\" ":{"labels":[{"name":"database_name","value":"dunhuang"}],"samples":[{"value":4483710976,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"games@002dlauncher@002danalysis\" ":{"labels":[{"name":"database_name","value":"games@002dlauncher@002danalysis"}],"samples":[{"value":311296,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"games_launcher\" ":{"labels":[{"name":"database_name","value":"games_launcher"}],"samples":[{"value":31309824,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter"}],"samples":[{"value":152535040,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_cq4\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_cq4"}],"samples":[{"value":83312640,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_cq5\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_cq5"}],"samples":[{"value":91766784,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_nj6\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_nj6"}],"samples":[{"value":83378176,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_sz3\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_sz3"}],"samples":[{"value":108412928,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_tj4\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_tj4"}],"samples":[{"value":48627712,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_configcenter_tj7\" ":{"labels":[{"name":"database_name","value":"gmve_configcenter_tj7"}],"samples":[{"value":79052800,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_license\" ":{"labels":[{"name":"database_name","value":"gmve_license"}],"samples":[{"value":294912,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"gmve_license_center\" ":{"labels":[{"name":"database_name","value":"gmve_license_center"}],"samples":[{"value":344064,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"grafana\" ":{"labels":[{"name":"database_name","value":"grafana"}],"samples":[{"value":61587456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"greatwall\" ":{"labels":[{"name":"database_name","value":"greatwall"}],"samples":[{"value":1310720,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"offline\" ":{"labels":[{"name":"database_name","value":"offline"}],"samples":[{"value":70555746304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcg_backend_dev\" ":{"labels":[{"name":"database_name","value":"opcg_backend_dev"}],"samples":[{"value":27525120,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcg_backend_pro\" ":{"labels":[{"name":"database_name","value":"opcg_backend_pro"}],"samples":[{"value":3726671872,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcg_cluster\" ":{"labels":[{"name":"database_name","value":"opcg_cluster"}],"samples":[{"value":20987904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcg_cluster_test\" ":{"labels":[{"name":"database_name","value":"opcg_cluster_test"}],"samples":[{"value":33882112,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcg_db_hub\" ":{"labels":[{"name":"database_name","value":"opcg_db_hub"}],"samples":[{"value":2855911424,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"opcgvirt_gmatrix\" ":{"labels":[{"name":"database_name","value":"opcgvirt_gmatrix"}],"samples":[{"value":170098688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"paladin\" ":{"labels":[{"name":"database_name","value":"paladin"}],"samples":[{"value":11010048,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"qa1_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"qa1_gmve_configcenter"}],"samples":[{"value":43220992,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"qa2_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"qa2_gmve_configcenter"}],"samples":[{"value":104644608,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"rc_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"rc_gmve_configcenter"}],"samples":[{"value":25280512,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"scrm\" ":{"labels":[{"name":"database_name","value":"scrm"}],"samples":[{"value":248476893184,"timestamp":1733068800000},{"value":248476893184,"timestamp":1733155200000},{"value":248476893184,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"sr_server\" ":{"labels":[{"name":"database_name","value":"sr_server"}],"samples":[{"value":17973248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"sr_server_qa2\" ":{"labels":[{"name":"database_name","value":"sr_server_qa2"}],"samples":[{"value":30343168,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"sr_server_rc2\" ":{"labels":[{"name":"database_name","value":"sr_server_rc2"}],"samples":[{"value":1163264,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"sr_server_test\" ":{"labels":[{"name":"database_name","value":"sr_server_test"}],"samples":[{"value":5652480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"tdw_export\" ":{"labels":[{"name":"database_name","value":"tdw_export"}],"samples":[{"value":311296,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"test\" ":{"labels":[{"name":"database_name","value":"test"}],"samples":[{"value":32702464,"timestamp":1733068800000},{"value":32702464,"timestamp":1733155200000},{"value":32702464,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"test_db\" ":{"labels":[{"name":"database_name","value":"test_db"}],"samples":[{"value":114688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"test_gmve_recorder\" ":{"labels":[{"name":"database_name","value":"test_gmve_recorder"}],"samples":[{"value":557056,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"test_tv_backend_db\" ":{"labels":[{"name":"database_name","value":"test_tv_backend_db"}],"samples":[{"value":1327104,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"time_controller_prod\" ":{"labels":[{"name":"database_name","value":"time_controller_prod"}],"samples":[{"value":1905320722432,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"tiyan_gmve_recorder\" ":{"labels":[{"name":"database_name","value":"tiyan_gmve_recorder"}],"samples":[{"value":229376,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"tmp_gmve_configcenter\" ":{"labels":[{"name":"database_name","value":"tmp_gmve_configcenter"}],"samples":[{"value":10944512,"timestamp":1733241600000}],"exemplars":null,"histograms":null},"name:\"database_name\" value:\"zhiqiangli_test\" ":{"labels":[{"name":"database_name","value":"zhiqiangli_test"}],"samples":[{"value":819200,"timestamp":1733241600000}],"exemplars":null,"histograms":null}}`, + expected: `{"timeseries":[{"labels":[{"name":"database_name","value":"dbRaphael"}],"samples":[{"value":649154281472,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"bak_cbs_dbRaphael_new_20220117170335_100630794"}],"samples":[{"value":917192704,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dbRaphael_new"}],"samples":[{"value":917192704,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"time_controller_prod"}],"samples":[{"value":1905320722432,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_time"}],"samples":[{"value":158793728,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_gamerapp"}],"samples":[{"value":165445632,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"analysis"}],"samples":[{"value":109520945152,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"scrm"}],"samples":[{"value":248476893184,"timestamp":1733068800000},{"value":248476893184,"timestamp":1733155200000},{"value":248476893184,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dbRaphaelBiz"}],"samples":[{"value":64331317248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"boss"}],"samples":[{"value":98140160,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_gamematrix_web_manage_prod"}],"samples":[{"value":3144843264,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcg_backend_dev"}],"samples":[{"value":27525120,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter"}],"samples":[{"value":152535040,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcg_backend_pro"}],"samples":[{"value":3726671872,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_task"}],"samples":[{"value":199426048,"timestamp":1733068800000},{"value":199426048,"timestamp":1733155200000},{"value":199426048,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_config"}],"samples":[{"value":620756992,"timestamp":1733068800000},{"value":620756992,"timestamp":1733155200000},{"value":620756992,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_ticket"}],"samples":[{"value":5230510080,"timestamp":1733068800000},{"value":5238898688,"timestamp":1733155200000},{"value":5251481600,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_mc"}],"samples":[{"value":130160410624,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"grafana"}],"samples":[{"value":61587456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_customer"}],"samples":[{"value":10132996096,"timestamp":1733068800000},{"value":10132996096,"timestamp":1733155200000},{"value":10132996096,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_xinyue_robot"}],"samples":[{"value":3490217984,"timestamp":1733068800000},{"value":3490217984,"timestamp":1733155200000},{"value":3490217984,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_cloudgames"}],"samples":[{"value":17596563456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_scpal"}],"samples":[{"value":10747904,"timestamp":1733068800000},{"value":10747904,"timestamp":1733155200000},{"value":10747904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_activity"}],"samples":[{"value":2097152,"timestamp":1733068800000},{"value":2097152,"timestamp":1733155200000},{"value":2097152,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_im"}],"samples":[{"value":818462720,"timestamp":1733068800000},{"value":830914560,"timestamp":1733155200000},{"value":830914560,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcg_cluster"}],"samples":[{"value":20987904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcg_cluster_test"}],"samples":[{"value":33882112,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"offline"}],"samples":[{"value":70555746304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_backup"}],"samples":[{"value":20854243328,"timestamp":1733068800000},{"value":20359315456,"timestamp":1733155200000},{"value":20455784448,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_live_act"}],"samples":[{"value":1343488,"timestamp":1733068800000},{"value":1343488,"timestamp":1733155200000},{"value":1343488,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_svip_data_flow"}],"samples":[{"value":420724736,"timestamp":1733068800000},{"value":420724736,"timestamp":1733155200000},{"value":420724736,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_h5_backend"}],"samples":[{"value":145408000,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"sr_server"}],"samples":[{"value":17973248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_mp"}],"samples":[{"value":1064960,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"games_launcher"}],"samples":[{"value":31309824,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_external_idata"}],"samples":[{"value":4674715648,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_robot_platform"}],"samples":[{"value":118902964224,"timestamp":1733068800000},{"value":118902964224,"timestamp":1733155200000},{"value":118902964224,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_self_service"}],"samples":[{"value":52723712,"timestamp":1733068800000},{"value":52723712,"timestamp":1733155200000},{"value":52723712,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"test_tv_backend_db"}],"samples":[{"value":1327104,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_ocpauth"}],"samples":[{"value":1589248,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_cq4"}],"samples":[{"value":83312640,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_cq5"}],"samples":[{"value":91766784,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_nj6"}],"samples":[{"value":83378176,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_sz3"}],"samples":[{"value":108412928,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_tj4"}],"samples":[{"value":48627712,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_configcenter_tj7"}],"samples":[{"value":79052800,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"deploy_gmve_configcenter"}],"samples":[{"value":1556480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcg_db_hub"}],"samples":[{"value":2855911424,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"paladin"}],"samples":[{"value":11010048,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"tmp_gmve_configcenter"}],"samples":[{"value":10944512,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"cloudgame"}],"samples":[{"value":83951616,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dev_gmve_configcenter"}],"samples":[{"value":12713984,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"qa1_gmve_configcenter"}],"samples":[{"value":43220992,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"rc_gmve_configcenter"}],"samples":[{"value":25280512,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dbXyplusService"}],"samples":[{"value":110100480,"timestamp":1733068800000},{"value":110100480,"timestamp":1733155200000},{"value":110100480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_live_vote"}],"samples":[{"value":458752,"timestamp":1733068800000},{"value":458752,"timestamp":1733155200000},{"value":458752,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_team"}],"samples":[{"value":950272,"timestamp":1733068800000},{"value":950272,"timestamp":1733155200000},{"value":950272,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_approval"}],"samples":[{"value":11845632,"timestamp":1733068800000},{"value":11845632,"timestamp":1733155200000},{"value":11845632,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_satisfaction"}],"samples":[{"value":376832,"timestamp":1733068800000},{"value":376832,"timestamp":1733155200000},{"value":376832,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_voice"}],"samples":[{"value":1343488,"timestamp":1733068800000},{"value":1343488,"timestamp":1733155200000},{"value":1343488,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_xinyue_robot_act"}],"samples":[{"value":20178796544,"timestamp":1733068800000},{"value":20178796544,"timestamp":1733155200000},{"value":20178796544,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"qa2_gmve_configcenter"}],"samples":[{"value":104644608,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"test"}],"samples":[{"value":32702464,"timestamp":1733068800000},{"value":32702464,"timestamp":1733155200000},{"value":32702464,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"sr_server_qa2"}],"samples":[{"value":30343168,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"zhiqiangli_test"}],"samples":[{"value":819200,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"sr_server_rc2"}],"samples":[{"value":1163264,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_danmu"}],"samples":[{"value":1835008,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_svip_station"}],"samples":[{"value":278528,"timestamp":1733068800000},{"value":278528,"timestamp":1733155200000},{"value":278528,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_license_center"}],"samples":[{"value":344064,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"data"}],"samples":[{"value":46137344,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_inner_idata"}],"samples":[{"value":32751616,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"opcgvirt_gmatrix"}],"samples":[{"value":170098688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"digitalgw"}],"samples":[{"value":1333788672,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_game"}],"samples":[{"value":98304,"timestamp":1733068800000},{"value":98304,"timestamp":1733155200000},{"value":98304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_hostPasswd"}],"samples":[{"value":77709312,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_robot_oper_closedloop"}],"samples":[{"value":104562688,"timestamp":1733068800000},{"value":104562688,"timestamp":1733155200000},{"value":104562688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_user"}],"samples":[{"value":163840,"timestamp":1733068800000},{"value":163840,"timestamp":1733155200000},{"value":163840,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_vdesk_llm"}],"samples":[{"value":98304,"timestamp":1733068800000},{"value":98304,"timestamp":1733155200000},{"value":98304,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"db_xinyue_robot_log"}],"samples":[{"value":20356399104,"timestamp":1733068800000},{"value":20356399104,"timestamp":1733155200000},{"value":20356399104,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dbtest"}],"samples":[{"value":114688,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"games@002dlauncher@002danalysis"}],"samples":[{"value":311296,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"gmve_license"}],"samples":[{"value":294912,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"greatwall"}],"samples":[{"value":1310720,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"codeaxis"}],"samples":[{"value":9584640,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dev_digitalgw"}],"samples":[{"value":507904,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dev_dunhuang"}],"samples":[{"value":294912,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dev_gmve_recorder"}],"samples":[{"value":458752,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"dunhuang"}],"samples":[{"value":4483710976,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"sr_server_test"}],"samples":[{"value":5652480,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"tdw_export"}],"samples":[{"value":311296,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"test_gmve_recorder"}],"samples":[{"value":557056,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"tiyan_gmve_recorder"}],"samples":[{"value":229376,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"bak_cbs_cloudgame_201911131726"}],"samples":[{"value":147456,"timestamp":1733241600000}],"exemplars":null,"histograms":null},{"labels":[{"name":"database_name","value":"test_db"}],"samples":[{"value":114688,"timestamp":1733241600000}],"exemplars":null,"histograms":null}]}`, }, } diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index 03b36376c..c4f7911da 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -21,7 +21,6 @@ import ( elastic "github.com/olivere/elastic/v7" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" @@ -377,16 +376,13 @@ func (i *Instance) esQuery(ctx context.Context, qo *queryOption, fact *FormatFac return res, err } -func (i *Instance) queryWithAgg(ctx context.Context, qo *queryOption, fact *FormatFactory, rets chan<- *TimeSeriesResult) { +func (i *Instance) queryWithAgg(ctx context.Context, qo *queryOption, fact *FormatFactory) storage.SeriesSet { var ( - ret = TimeSeriesResultPool.Get().(*TimeSeriesResult) err error ) ctx, span := trace.NewSpan(ctx, "query-with-aggregation") defer func() { span.End(&err) - ret.Error = err - rets <- ret }() span.Set("query-conn", qo.conn) @@ -395,102 +391,16 @@ func (i *Instance) queryWithAgg(ctx context.Context, qo *queryOption, fact *Form sr, err := i.esQuery(ctx, qo, fact) if err != nil { - return + return storage.ErrSeriesSet(err) } // 如果是非时间聚合计算，则无需进行指标名的拼接作用 - ret.TimeSeriesMap, err = fact.AggDataFormat(sr.Aggregations, metricLabel) - if err != nil { - return - } - return -} - -func (i *Instance) queryWithoutAgg(ctx context.Context, qo *queryOption, fact *FormatFactory, rets chan<- *TimeSeriesResult) { - var ( - ret = TimeSeriesResultPool.Get().(*TimeSeriesResult) - err error - ) - ctx, span := trace.NewSpan(ctx, "query-without-aggregation") - defer func() { - span.End(&err) - ret.Error = err - rets <- ret - }() - - sr, err := i.esQuery(ctx, qo, fact) + qr, err := fact.AggDataFormat(sr.Aggregations, metricLabel) if err != nil { - return - } - - ret.TimeSeriesMap = make(map[string]*prompb.TimeSeries) - for _, d := range sr.Hits.Hits { - data := make(map[string]interface{}) - if err = json.Unmarshal(d.Source, &data); err != nil { - return - } - fact.SetData(data) - - lbs, vErr := fact.Labels() - if vErr != nil { - err = vErr - return - } - - sample, vErr := fact.Sample() - if vErr != nil { - err = vErr - return - } - - if _, ok := ret.TimeSeriesMap[lbs.String()]; !ok { - ret.TimeSeriesMap[lbs.String()] = &prompb.TimeSeries{ - Labels: lbs.GetLabels(), - Samples: make([]prompb.Sample, 0), - } - } - ret.TimeSeriesMap[lbs.String()].Samples = append(ret.TimeSeriesMap[lbs.String()].Samples, sample) - } - - return -} - -func (i *Instance) mergeTimeSeries(rets chan *TimeSeriesResult, mergeFunc func(...[]prompb.Sample) []prompb.Sample) (*prompb.QueryResult, error) { - seriesMap := make(map[string]*prompb.TimeSeries) - - for ret := range rets { - if ret.Error != nil { - return nil, ret.Error - } - - if len(ret.TimeSeriesMap) == 0 { - continue - } - - for key, ts := range ret.TimeSeriesMap { - if _, ok := seriesMap[key]; !ok { - seriesMap[key] = &prompb.TimeSeries{ - Labels: ts.GetLabels(), - Samples: make([]prompb.Sample, 0), - } - } - - seriesMap[key].Samples = mergeFunc(seriesMap[key].Samples, ts.Samples) - } - - ret.TimeSeriesMap = nil - ret.Error = nil - TimeSeriesResultPool.Put(ret) - } - - qr := &prompb.QueryResult{ - Timeseries: make([]*prompb.TimeSeries, 0, len(seriesMap)), - } - for _, ts := range seriesMap { - qr.Timeseries = append(qr.Timeseries, ts) + return storage.ErrSeriesSet(err) } - return qr, nil + return remote.FromQueryResult(false, qr) } func (i *Instance) getAlias(ctx context.Context, db string, needAddTime bool, start, end time.Time, timezone string) ([]string, error) { @@ -768,26 +678,21 @@ func (i *Instance) QuerySeriesSet( span.Set("query-field", query.Field) span.Set("query-fields", query.Fields) - rets := make(chan *TimeSeriesResult, 1) + setCh := make(chan storage.SeriesSet, len(i.connects)) go func() { defer func() { // es 查询有很多结构体无法判断的，会导致 panic if r := recover(); r != nil { - rets <- &TimeSeriesResult{ - Error: fmt.Errorf("es query error: %s", r), - } + setCh <- storage.ErrSeriesSet(fmt.Errorf("es query error: %s", r)) } - close(rets) + close(setCh) }() aliases, err1 := i.getAlias(ctx, query.DB, query.NeedAddTime, start, end, query.Timezone) if err1 != nil { - rets <- &TimeSeriesResult{ - Error: err1, - } - return + setCh <- storage.ErrSeriesSet(err1) } span.Set("query-aliases", aliases) @@ -816,9 +721,7 @@ func (i *Instance) QuerySeriesSet( } if err1 != nil { - rets <- &TimeSeriesResult{ - Error: err1, - } + setCh <- storage.ErrSeriesSet(err1) return } var size int @@ -835,28 +738,26 @@ func (i *Instance) QuerySeriesSet( WithOrders(query.Orders). WithTransform(metadata.GetPromDataFormat(ctx).EncodeFunc(), metadata.GetPromDataFormat(ctx).DecodeFunc()) - if len(query.Aggregates) > 0 { - i.queryWithAgg(ctx, qo, fact, rets) - } else { - i.queryWithoutAgg(ctx, qo, fact, rets) + if len(query.Aggregates) == 0 { + setCh <- storage.ErrSeriesSet(fmt.Errorf("aggregates is empty")) + return } + setCh <- i.queryWithAgg(ctx, qo, fact) }() } wg.Wait() }() - qr, err := i.mergeTimeSeries(rets, function.MergeSamplesWithFuncAndSort(query.Aggregates.LastAggName())) - if err != nil { - return storage.ErrSeriesSet(err) - } - - if qr == nil || len(qr.Timeseries) == 0 { - return storage.EmptySeriesSet() + var sets []storage.SeriesSet + for s := range setCh { + if s != nil { + sets = append(sets, s) + } } - return remote.FromQueryResult(true, qr) + return storage.NewMergeSeriesSet(sets, function.NewMergeSeriesSetWithFuncAndSort(query.Aggregates.LastAggName())) } // QueryRange 使用 es 直接查询引擎 diff --git a/pkg/unify-query/tsdb/prometheus/querier.go b/pkg/unify-query/tsdb/prometheus/querier.go index 99cdf087b..c8b732348 100644 --- a/pkg/unify-query/tsdb/prometheus/querier.go +++ b/pkg/unify-query/tsdb/prometheus/querier.go @@ -126,7 +126,7 @@ func (q *Querier) selectFn(hints *storage.SelectHints, matchers ...*labels.Match } } - set = storage.NewMergeSeriesSet(sets, NewMergeSeriesSetWithFuncAndSort(hints.Func)) + set = storage.NewMergeSeriesSet(sets, function.NewMergeSeriesSetWithFuncAndSort(hints.Func)) }() for _, m := range matchers { From 0f698b954e47212b43ba2f7736cbfcbbd43210c6 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 24 Apr 2025 20:32:18 +0800 Subject: [PATCH 098/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 61 ++++++++++++++++-- pkg/unify-query/tsdb/bksql/instance.go | 4 ++ pkg/unify-query/tsdb/bksql/instance_test.go | 42 ++++++++++++- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 12 ++++ pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 68 +++++++++++++++++---- 5 files changed, 169 insertions(+), 18 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 9cd371751..c9bffc5ac 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -68,12 +68,15 @@ type QueryFactory struct { timeField string expr sqlExpr.SQLExpr + + highlight *metadata.HighLight } func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f := &QueryFactory{ - ctx: ctx, - query: query, + ctx: ctx, + query: query, + highlight: query.HighLight, } if query.Orders != nil { @@ -89,6 +92,11 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f.expr = sqlExpr.GetSQLExpr(f.query.Measurement). WithInternalFields(f.timeField, query.Field). WithEncode(metadata.GetPromDataFormat(ctx).EncodeFunc()) + + if f.highlight != nil && f.highlight.Enable { + f.expr.IsSetLabels(true) + } + return f } @@ -124,10 +132,53 @@ func (f *QueryFactory) FieldMap() map[string]string { return f.expr.FieldMap() } -func (f *QueryFactory) ReloadListData(data map[string]any) map[string]any { - newData := make(map[string]any) +func (f *QueryFactory) HighLight(data map[string]any) (newData map[string]any) { + if f.query.HighLight == nil || !f.query.HighLight.Enable { + return + } + + newData = make(map[string]any) + for k, vs := range f.expr.GetLabelMap() { + if vs == nil { + return + } + + if d, ok := data[k]; ok { + var ( + mark1 string + mark2 string + ) + + switch s := d.(type) { + case string: + if f.query.HighLight.MaxAnalyzedOffset > 0 && len(s) > f.query.HighLight.MaxAnalyzedOffset { + mark1 = s[0:f.query.HighLight.MaxAnalyzedOffset] + mark2 = s[f.query.HighLight.MaxAnalyzedOffset:] + } else { + mark1 = s + } + + for _, v := range vs { + mark1 = strings.ReplaceAll(mark1, v, fmt.Sprintf("%s", v)) + } + + res := fmt.Sprintf("%s%s", mark1, mark2) + if res != d { + newData[k] = []string{res} + } + } + + } + } + + return +} + +func (f *QueryFactory) ReloadListData(data map[string]any) (newData map[string]any) { + newData = make(map[string]any) fieldMap := f.FieldMap() + for k, d := range data { if v, ok := fieldMap[k]; ok { if v == TableTypeVariant { @@ -145,7 +196,7 @@ func (f *QueryFactory) ReloadListData(data map[string]any) map[string]any { newData[k] = d } - return newData + return } func (f *QueryFactory) FormatDataToQueryResult(ctx context.Context, list []map[string]interface{}) (*prompb.QueryResult, error) { diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 6f368b629..2f48d34fb 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -311,6 +311,10 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star newData[KeyIndex] = query.DB newData[KeyTableID] = query.TableID newData[KeyDataLabel] = query.DataLabel + + if query.HighLight != nil && query.HighLight.Enable { + newData[KeyHighLight] = queryFactory.HighLight(newData) + } dataCh <- newData } diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 9ff183d27..240adddc4 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -240,8 +240,14 @@ func TestInstance_QueryRaw(t *testing.T) { mock.BkSQL.Set(map[string]any{ "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, + // query with in "SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `132_lol_new_login_queue_login_1min` WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"132_lol_new_login_queue_login_1min\":{}},\"cluster\":\"default2\",\"totalRecords\":5,\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:31:00\",\"dtEventTimeStamp\":1730118660000,\"localTime\":\"2024-10-28 20:32:03\",\"_startTime_\":\"2024-10-28 20:31:00\",\"_endTime_\":\"2024-10-28 20:32:00\",\"namespace\":\"gz100\",\"login_rate\":269.0,\"_value_\":269.0,\"_timestamp_\":1730118660000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:28:00\",\"dtEventTimeStamp\":1730118480000,\"localTime\":\"2024-10-28 20:29:03\",\"_startTime_\":\"2024-10-28 20:28:00\",\"_endTime_\":\"2024-10-28 20:29:00\",\"namespace\":\"gz100\",\"login_rate\":271.0,\"_value_\":271.0,\"_timestamp_\":1730118480000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:29:00\",\"dtEventTimeStamp\":1730118540000,\"localTime\":\"2024-10-28 20:30:02\",\"_startTime_\":\"2024-10-28 20:29:00\",\"_endTime_\":\"2024-10-28 20:30:00\",\"namespace\":\"gz100\",\"login_rate\":267.0,\"_value_\":267.0,\"_timestamp_\":1730118540000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:30:00\",\"dtEventTimeStamp\":1730118600000,\"localTime\":\"2024-10-28 20:31:04\",\"_startTime_\":\"2024-10-28 20:30:00\",\"_endTime_\":\"2024-10-28 20:31:00\",\"namespace\":\"gz100\",\"login_rate\":274.0,\"_value_\":274.0,\"_timestamp_\":1730118600000},{\"thedate\":20241028,\"dtEventTime\":\"2024-10-28 20:27:00\",\"dtEventTimeStamp\":1730118420000,\"localTime\":\"2024-10-28 20:28:03\",\"_startTime_\":\"2024-10-28 20:27:00\",\"_endTime_\":\"2024-10-28 20:28:00\",\"namespace\":\"gz100\",\"login_rate\":279.0,\"_value_\":279.0,\"_timestamp_\":1730118420000}],\"select_fields_order\":[\"thedate\",\"dtEventTime\",\"dtEventTimeStamp\",\"localTime\",\"_startTime_\",\"_endTime_\",\"namespace\",\"login_rate\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `login_rate` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_132.lol_new_login_queue_login_1min_132 WHERE ((`dtEventTimeStamp` >= 1730118415782) AND (`dtEventTimeStamp` < 1730118715782)) AND `namespace` IN ('gz100', 'bgp2-new') LIMIT 10005\",\"total_record_size\":5832,\"timetaken\":0.251,\"bksql_call_elapsed_time\":0,\"device\":\"tspider\",\"result_table_ids\":[\"132_lol_new_login_queue_login_1min\"]},\"errors\":null,\"trace_id\":\"c083ca92cee435138f9076e1c1f6faeb\",\"span_id\":\"735f314a259a981a\"}", - "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + + // query raw by doris + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", + + // query raw by doris and highlight + "SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `5000140_bklog_container_log_demo_analysis`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND (`log` MATCH_PHRASE_PREFIX 'metricbeat_scrape' OR `log` MATCH_PHRASE_PREFIX 'metricbeat') LIMIT 2": "{\"result\":true,\"message\":\"成功\",\"code\":\"00\",\"data\":{\"result_table_scan_range\":{\"5000140_bklog_container_log_demo_analysis\":{\"start\":\"2025032100\",\"end\":\"2025032123\"}},\"cluster\":\"doris_bklog\",\"totalRecords\":2,\"external_api_call_time_mills\":{\"bkbase_meta_api\":0},\"resource_use_summary\":{\"cpu_time_mills\":0,\"memory_bytes\":0,\"processed_bytes\":0,\"processed_rows\":0},\"source\":\"\",\"list\":[{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":2,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000},{\"thedate\":20250321,\"dteventtimestamp\":1742540043000,\"dteventtime\":\"2025-03-21 14:54:03\",\"localtime\":\"2025-03-21 14:54:12\",\"__shard_key__\":29042334000,\"_starttime_\":\"2025-03-21 14:54:03\",\"_endtime_\":\"2025-03-21 14:54:03\",\"bk_host_id\":267382,\"__ext\":\"{\\\"container_id\\\":\\\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\\\",\\\"container_image\\\":\\\"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf\\\",\\\"container_name\\\":\\\"bkmonitorbeat\\\",\\\"io_kubernetes_pod\\\":\\\"bkm-daemonset-worker-9tckj\\\",\\\"io_kubernetes_pod_namespace\\\":\\\"bkmonitor-operator\\\",\\\"io_kubernetes_pod_uid\\\":\\\"0d310b8f-aca1-48ab-b02c-92f5c221eac3\\\",\\\"io_kubernetes_workload_name\\\":\\\"bkm-daemonset-worker\\\",\\\"io_kubernetes_workload_type\\\":\\\"DaemonSet\\\",\\\"labels\\\":{\\\"app_kubernetes_io_component\\\":\\\"bkmonitorbeat\\\",\\\"controller_revision_hash\\\":\\\"6b87cb95fc\\\",\\\"pod_template_generation\\\":\\\"14\\\"}}\",\"cloudid\":0,\"path\":\"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log\",\"gseindex\":7451424,\"iterationindex\":1,\"log\":\"2025-03-21 06:54:03.766\\tINFO\\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]\",\"logtime\":\"2025-03-21 06:54:03.766\",\"level\":\"INFO\",\"cid\":\"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e\",\"time\":1742540045,\"_value_\":267382,\"_timestamp_\":1742540043000}],\"stage_elapsed_time_mills\":{\"check_query_syntax\":1,\"query_db\":21,\"get_query_driver\":0,\"match_query_forbidden_config\":0,\"convert_query_statement\":1,\"connect_db\":45,\"match_query_routing_rule\":0,\"check_permission\":0,\"check_query_semantic\":0,\"pick_valid_storage\":1},\"select_fields_order\":[\"thedate\",\"dteventtimestamp\",\"dteventtime\",\"localtime\",\"__shard_key__\",\"_starttime_\",\"_endtime_\",\"bk_host_id\",\"__ext\",\"cloudid\",\"serverip\",\"path\",\"gseindex\",\"iterationindex\",\"log\",\"logtime\",\"level\",\"cid\",\"time\",\"_value_\",\"_timestamp_\"],\"sql\":\"SELECT *, `bk_host_id` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2 WHERE `thedate` = '20250321' LIMIT 2\",\"total_record_size\":9304,\"timetaken\":0.069,\"result_schema\":[{\"field_type\":\"int\",\"field_name\":\"__c0\",\"field_alias\":\"thedate\",\"field_index\":0},{\"field_type\":\"long\",\"field_name\":\"__c1\",\"field_alias\":\"dteventtimestamp\",\"field_index\":1},{\"field_type\":\"string\",\"field_name\":\"__c2\",\"field_alias\":\"dteventtime\",\"field_index\":2},{\"field_type\":\"string\",\"field_name\":\"__c3\",\"field_alias\":\"localtime\",\"field_index\":3},{\"field_type\":\"long\",\"field_name\":\"__c4\",\"field_alias\":\"__shard_key__\",\"field_index\":4},{\"field_type\":\"string\",\"field_name\":\"__c5\",\"field_alias\":\"_starttime_\",\"field_index\":5},{\"field_type\":\"string\",\"field_name\":\"__c6\",\"field_alias\":\"_endtime_\",\"field_index\":6},{\"field_type\":\"int\",\"field_name\":\"__c7\",\"field_alias\":\"bk_host_id\",\"field_index\":7},{\"field_type\":\"string\",\"field_name\":\"__c8\",\"field_alias\":\"__ext\",\"field_index\":8},{\"field_type\":\"int\",\"field_name\":\"__c9\",\"field_alias\":\"cloudid\",\"field_index\":9},{\"field_type\":\"string\",\"field_name\":\"__c10\",\"field_alias\":\"serverip\",\"field_index\":10},{\"field_type\":\"string\",\"field_name\":\"__c11\",\"field_alias\":\"path\",\"field_index\":11},{\"field_type\":\"long\",\"field_name\":\"__c12\",\"field_alias\":\"gseindex\",\"field_index\":12},{\"field_type\":\"int\",\"field_name\":\"__c13\",\"field_alias\":\"iterationindex\",\"field_index\":13},{\"field_type\":\"string\",\"field_name\":\"__c14\",\"field_alias\":\"log\",\"field_index\":14},{\"field_type\":\"string\",\"field_name\":\"__c15\",\"field_alias\":\"logtime\",\"field_index\":15},{\"field_type\":\"string\",\"field_name\":\"__c16\",\"field_alias\":\"level\",\"field_index\":16},{\"field_type\":\"string\",\"field_name\":\"__c17\",\"field_alias\":\"cid\",\"field_index\":17},{\"field_type\":\"long\",\"field_name\":\"__c18\",\"field_alias\":\"time\",\"field_index\":18},{\"field_type\":\"int\",\"field_name\":\"__c19\",\"field_alias\":\"_value_\",\"field_index\":19},{\"field_type\":\"long\",\"field_name\":\"__c20\",\"field_alias\":\"_timestamp_\",\"field_index\":20}],\"bksql_call_elapsed_time\":0,\"device\":\"doris\",\"result_table_ids\":[\"5000140_bklog_container_log_demo_analysis\"]},\"errors\":null,\"trace_id\":\"1d6580ef7e6d7e7c040801a72645fdf2\",\"span_id\":\"ab5485e1dd6595bc\"}", }) end := time.UnixMilli(1730118889181) @@ -289,6 +295,40 @@ func TestInstance_QueryRaw(t *testing.T) { }, expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, }, + "query raw by doris use querystring in highlight": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + QueryString: "metricbeat_scrape metricbeat", + Size: 2, + HighLight: &metadata.HighLight{ + MaxAnalyzedOffset: 0, + Enable: true, + }, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, + "query raw by doris use querystring in highlight and max analyzed offset is 40": { + query: &metadata.Query{ + TableID: "5000140_bklog_container_log_demo_analysis.doris", + DB: "5000140_bklog_container_log_demo_analysis", + Measurement: "doris", + MetricName: "bk_host_id", + Field: "bk_host_id", + DataLabel: "5000140_bklog_container_log_demo_analysis", + QueryString: "metricbeat_scrape metricbeat", + Size: 2, + HighLight: &metadata.HighLight{ + MaxAnalyzedOffset: 40, + Enable: true, + }, + }, + expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, + }, } { t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index 09ff20a1a..cfb2d9001 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -50,6 +50,8 @@ type SQLExpr interface { WithEncode(func(string) string) SQLExpr // WithInternalFields 设置内部字段 WithInternalFields(timeField, valueField string) SQLExpr + // IsSetLabels 是否保留查询标签 + IsSetLabels(bool) SQLExpr // ParserQueryString 解析 es 特殊语法 queryString 生成SQL条件 ParserQueryString(qs string) (string, error) // ParserAllConditions 解析全量条件生成SQL条件表达式 @@ -60,6 +62,8 @@ type SQLExpr interface { DescribeTableSQL(table string) string // FieldMap 返回当前表结构 FieldMap() map[string]string + // GetLabelMap 返回当前查询值 + GetLabelMap() map[string][]string // Type 返回表达式类型 Type() string } @@ -128,6 +132,10 @@ func (d *DefaultSQLExpr) Type() string { return "default" } +func (d *DefaultSQLExpr) IsSetLabels(_ bool) SQLExpr { + return d +} + func (d *DefaultSQLExpr) WithInternalFields(timeField, valueField string) SQLExpr { d.timeField = timeField d.valueField = valueField @@ -149,6 +157,10 @@ func (d *DefaultSQLExpr) WithKeepColumns(cols []string) SQLExpr { return d } +func (d *DefaultSQLExpr) GetLabelMap() map[string][]string { + return nil +} + func (d *DefaultSQLExpr) FieldMap() map[string]string { return d.fieldMap } diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 863b79a7d..7ac02e43e 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -12,6 +12,7 @@ package sqlExpr import ( "fmt" "strings" + "sync" "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/internal/querystring" @@ -35,6 +36,11 @@ type DorisSQLExpr struct { keepColumns []string fieldsMap map[string]string + + isSetLabels bool + lock sync.Mutex + labelCheck map[string]struct{} + labelMap map[string][]string } var _ SQLExpr = (*DorisSQLExpr)(nil) @@ -54,6 +60,11 @@ func (d *DorisSQLExpr) WithEncode(fn func(string) string) SQLExpr { return d } +func (d *DorisSQLExpr) IsSetLabels(isSetLabels bool) SQLExpr { + d.isSetLabels = isSetLabels + return d +} + func (d *DorisSQLExpr) WithFieldsMap(fieldsMap map[string]string) SQLExpr { d.fieldsMap = fieldsMap return d @@ -68,6 +79,10 @@ func (d *DorisSQLExpr) FieldMap() map[string]string { return d.fieldsMap } +func (d *DorisSQLExpr) GetLabelMap() map[string][]string { + return d.labelMap +} + func (d *DorisSQLExpr) ParserQueryString(qs string) (string, error) { expr, err := querystring.Parse(qs) if err != nil { @@ -259,12 +274,14 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) } var ( - key string - op string - val string + oldKey string + key string + op string + val string ) - key, _ = d.dimTransform(c.DimensionName) + oldKey = c.DimensionName + key, _ = d.dimTransform(oldKey) // 对值进行转义处理 for i, v := range c.Value { @@ -276,8 +293,7 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) // 处理等于类操作符（=, IN, LIKE） case metadata.ConditionEqual, metadata.ConditionExact, metadata.ConditionContains: if len(c.Value) == 1 && c.Value[0] == "" { - op = "IS" - val = "NULL" + op = "IS NULL" break } @@ -302,6 +318,7 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) var filter []string for _, v := range c.Value { + d.addLabel(oldKey, v) filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) } key = "" @@ -313,8 +330,7 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) // 处理不等于类操作符（!=, NOT IN, NOT LIKE） case metadata.ConditionNotEqual, metadata.ConditionNotContains: if len(c.Value) == 1 && c.Value[0] == "" { - op = "IS NOT" - val = "NULL" + op = "IS NOT NULL" break } @@ -339,6 +355,9 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) var filter []string for _, v := range c.Value { + if v != "" { + d.addLabel(key, v) + } filter = append(filter, fmt.Sprintf("%s %s %s", key, op, fmt.Sprintf(format, v))) } key = "" @@ -384,7 +403,12 @@ func (d *DorisSQLExpr) buildCondition(c metadata.ConditionField) (string, error) } if key != "" { - return fmt.Sprintf("%s %s %s", key, op, val), nil + condition := fmt.Sprintf("%s %s", key, op) + if val != "" { + d.addLabel(oldKey, val) + condition = fmt.Sprintf("%s %s", condition, val) + } + return condition, nil } return val, nil } @@ -400,6 +424,27 @@ func (d *DorisSQLExpr) checkMatchALL(k string) bool { return false } +func (d *DorisSQLExpr) addLabel(key, value string) { + if !d.isSetLabels { + return + } + + d.lock.Lock() + defer d.lock.Unlock() + + if d.labelCheck == nil { + d.labelCheck = make(map[string]struct{}) + } + if d.labelMap == nil { + d.labelMap = make(map[string][]string) + } + + if _, ok := d.labelCheck[key+value]; !ok { + d.labelCheck[key+value] = struct{}{} + d.labelMap[key] = append(d.labelMap[key], value) + } +} + func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { var ( err error @@ -439,15 +484,15 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { c.Field = DefaultKey } + d.addLabel(c.Field, c.Value) field, _ := d.dimTransform(c.Field) - return fmt.Sprintf("%s LIKE '%%%s%%'", field, c.Value), nil case *querystring.MatchExpr: if c.Field == "" { c.Field = DefaultKey } + d.addLabel(c.Field, c.Value) field, _ := d.dimTransform(c.Field) - if d.checkMatchALL(c.Field) { return fmt.Sprintf("%s MATCH_PHRASE_PREFIX '%s'", field, c.Value), nil } @@ -458,7 +503,6 @@ func (d *DorisSQLExpr) walk(e querystring.Expr) (string, error) { c.Field = DefaultKey } field, _ := d.dimTransform(c.Field) - var timeFilter []string if c.Start != nil && *c.Start != "*" { var op string From b6124da0463a884fe77b781de663d41839ee762e Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 24 Apr 2025 21:32:30 +0800 Subject: [PATCH 099/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 6 ++++- pkg/unify-query/tsdb/bksql/instance.go | 7 ++---- pkg/unify-query/tsdb/bksql/instance_test.go | 28 +++++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index c9bffc5ac..42c8694a5 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -132,13 +132,17 @@ func (f *QueryFactory) FieldMap() map[string]string { return f.expr.FieldMap() } +func (f *QueryFactory) GetLabelMap() map[string][]string { + return f.expr.GetLabelMap() +} + func (f *QueryFactory) HighLight(data map[string]any) (newData map[string]any) { if f.query.HighLight == nil || !f.query.HighLight.Enable { return } newData = make(map[string]any) - for k, vs := range f.expr.GetLabelMap() { + for k, vs := range f.GetLabelMap() { if vs == nil { return } diff --git a/pkg/unify-query/tsdb/bksql/instance.go b/pkg/unify-query/tsdb/bksql/instance.go index 2f48d34fb..85773b8d4 100644 --- a/pkg/unify-query/tsdb/bksql/instance.go +++ b/pkg/unify-query/tsdb/bksql/instance.go @@ -299,12 +299,9 @@ func (i *Instance) QueryRawData(ctx context.Context, query *metadata.Query, star return } + span.Set("label-map", queryFactory.GetLabelMap()) span.Set("data-total-records", data.TotalRecords) - log.Infof(ctx, "total records: %d", data.TotalRecords) - - if i.maxLimit > 0 && data.TotalRecords > i.maxLimit { - return - } + span.Set("data-list-size", len(data.List)) for _, list := range data.List { newData := queryFactory.ReloadListData(list) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 240adddc4..5de17e8b1 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -238,6 +238,11 @@ func TestInstance_QueryRaw(t *testing.T) { ins := createTestInstance(ctx) mock.BkSQL.Set(map[string]any{ + // query raw by doris use condition in highlight + "SHOW CREATE TABLE `2_bklog_pure_v4_log_doris_for_unify_query`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris-test","totalRecords":18,"external_api_call_time_mills":{"bkbase_auth_api":69,"bkbase_meta_api":9,"bkbase_apigw_api":25},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"__shard_key__","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"cloudId","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"serverIp","Type":"varchar(512)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"path","Type":"varchar(512)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"gseIndex","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"iterationIndex","Type":"decimalv3(38, 6)","Null":"YES","Key":"YES","Default":null,"Extra":""},{"Field":"dtEventTimeStamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dtEventTime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localTime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"file","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"message","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"report_time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"trace_id","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":6,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":6,"connect_db":66,"match_query_routing_rule":0,"check_permission":69,"check_query_semantic":0,"pick_valid_storage":0},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_2.bklog_pure_v4_log_doris_for_unify_query_2","total_record_size":11808,"timetaken":0.148,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_pure_v4_log_doris_for_unify_query"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + // query raw by doris use condition in highlight + "SELECT *, `dtEventTimeStamp` AS `_value_`, `dtEventTimeStamp` AS `_timestamp_` FROM `2_bklog_pure_v4_log_doris_for_unify_query`.doris WHERE `dtEventTimeStamp` >= 1730118589181 AND `dtEventTimeStamp` <= 1730118889181 AND `thedate` = '20241028' AND (`message` MATCH_PHRASE_PREFIX 'Bk-Query-Source' OR (`level` MATCH_PHRASE_PREFIX 'error' OR `level` MATCH_PHRASE_PREFIX 'info')) LIMIT 5": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{"2_bklog_pure_v4_log_doris_for_unify_query":{"start":"2025042100","end":"2025042123"}},"cluster":"doris-test","totalRecords":5,"external_api_call_time_mills":{"bkbase_auth_api":38,"bkbase_meta_api":0,"bkbase_apigw_api":0},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":0.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.643Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.643Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":2.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.790Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.790Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":4.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:00.855Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:00.855Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":6.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:361","level":"info","log":"2025-04-21T11:25:01.030Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","report_time":"2025-04-21T11:25:01.030Z","time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5","_value_":1745234704000,"_timestamp_":1745234704000},{"thedate":20250421,"__shard_key__":29087245464,"cloudId":0.0,"path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","gseIndex":4281730.0,"iterationIndex":8.0,"dtEventTimeStamp":1745234704000,"dtEventTime":"2025-04-21 19:25:04","localTime":"2025-04-21 19:25:04","file":"http/handler.go:305","level":"info","log":"2025-04-21T11:25:01.199Z\tinfo\thttp/handler.go:305\t[adb84ecc380008245cdb800b6fd54d7f] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","report_time":"2025-04-21T11:25:01.199Z","time":"1745234704","trace_id":"adb84ecc380008245cdb800b6fd54d7f","_value_":1745234704000,"_timestamp_":1745234704000}],"stage_elapsed_time_mills":{"check_query_syntax":2,"query_db":19,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":2,"connect_db":41,"match_query_routing_rule":0,"check_permission":39,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["thedate","__shard_key__","cloudId","serverIp","path","gseIndex","iterationIndex","dtEventTimeStamp","dtEventTime","localTime","__ext","file","level","log","message","report_time","time","trace_id","_value_","_timestamp_"],"total_record_size":30880,"timetaken":0.104,"result_schema":[{"field_type":"int","field_name":"__c0","field_alias":"thedate","field_index":0},{"field_type":"long","field_name":"__c1","field_alias":"__shard_key__","field_index":1},{"field_type":"double","field_name":"__c2","field_alias":"cloudId","field_index":2},{"field_type":"string","field_name":"__c3","field_alias":"serverIp","field_index":3},{"field_type":"string","field_name":"__c4","field_alias":"path","field_index":4},{"field_type":"double","field_name":"__c5","field_alias":"gseIndex","field_index":5},{"field_type":"double","field_name":"__c6","field_alias":"iterationIndex","field_index":6},{"field_type":"long","field_name":"__c7","field_alias":"dtEventTimeStamp","field_index":7},{"field_type":"string","field_name":"__c8","field_alias":"dtEventTime","field_index":8},{"field_type":"string","field_name":"__c9","field_alias":"localTime","field_index":9},{"field_type":"string","field_name":"__c10","field_alias":"__ext","field_index":10},{"field_type":"string","field_name":"__c11","field_alias":"file","field_index":11},{"field_type":"string","field_name":"__c12","field_alias":"level","field_index":12},{"field_type":"string","field_name":"__c13","field_alias":"log","field_index":13},{"field_type":"string","field_name":"__c14","field_alias":"message","field_index":14},{"field_type":"string","field_name":"__c15","field_alias":"report_time","field_index":15},{"field_type":"string","field_name":"__c16","field_alias":"time","field_index":16},{"field_type":"string","field_name":"__c17","field_alias":"trace_id","field_index":17},{"field_type":"long","field_name":"__c18","field_alias":"_value_","field_index":18},{"field_type":"long","field_name":"__c19","field_alias":"_timestamp_","field_index":19}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["2_bklog_pure_v4_log_doris_for_unify_query"]},"errors":null,"trace_id":"00000000000000000000000000000000","span_id":"0000000000000000"}`, + "SHOW CREATE TABLE `5000140_bklog_container_log_demo_analysis`.doris": `{"result":true,"message":"成功","code":"00","data":{"result_table_scan_range":{},"cluster":"doris_bklog","totalRecords":19,"external_api_call_time_mills":{"bkbase_meta_api":10},"resource_use_summary":{"cpu_time_mills":0,"memory_bytes":0,"processed_bytes":0,"processed_rows":0},"source":"","list":[{"Field":"thedate","Type":"int","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtimestamp","Type":"bigint","Null":"NO","Key":"YES","Default":null,"Extra":""},{"Field":"dteventtime","Type":"varchar(32)","Null":"NO","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"localtime","Type":"varchar(32)","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__shard_key__","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_starttime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"_endtime_","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"bk_host_id","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"__ext","Type":"variant","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cloudid","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"serverip","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"path","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"gseindex","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"iterationindex","Type":"int","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"log","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"logtime","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"level","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"cid","Type":"text","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"},{"Field":"time","Type":"bigint","Null":"YES","Key":"NO","Default":null,"Extra":"NONE"}],"stage_elapsed_time_mills":{"check_query_syntax":1,"query_db":4,"get_query_driver":0,"match_query_forbidden_config":0,"convert_query_statement":10,"connect_db":43,"match_query_routing_rule":0,"check_permission":12,"check_query_semantic":0,"pick_valid_storage":1},"select_fields_order":["Field","Type","Null","Key","Default","Extra"],"sql":"SHOW COLUMNS FROM mapleleaf_5000140.bklog_container_log_demo_analysis_5000140__2","total_record_size":12408,"timetaken":0.071,"result_schema":[{"field_type":"string","field_name":"Field","field_alias":"Field","field_index":0},{"field_type":"string","field_name":"Type","field_alias":"Type","field_index":1},{"field_type":"string","field_name":"Null","field_alias":"Null","field_index":2},{"field_type":"string","field_name":"Key","field_alias":"Key","field_index":3},{"field_type":"string","field_name":"Default","field_alias":"Default","field_index":4},{"field_type":"string","field_name":"Extra","field_alias":"Extra","field_index":5}],"bksql_call_elapsed_time":0,"device":"doris","result_table_ids":["5000140_bklog_container_log_demo_analysis"]},"errors":null,"trace_id":"9ad04717f6b8da2c921d371cb3915cdf","span_id":"57ea24802f21b887"}`, // query with in @@ -329,6 +334,29 @@ func TestInstance_QueryRaw(t *testing.T) { }, expected: `[{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":2,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_line{} 274; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045},{"__data_label":"5000140_bklog_container_log_demo_analysis","__ext.container_id":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","__ext.container_image":"sha256:3aec083a12d24544c15f55559e80b571cb3e66e291c5f67f4366b0f9c75674bf","__ext.container_name":"bkmonitorbeat","__ext.io_kubernetes_pod":"bkm-daemonset-worker-9tckj","__ext.io_kubernetes_pod_namespace":"bkmonitor-operator","__ext.io_kubernetes_pod_uid":"0d310b8f-aca1-48ab-b02c-92f5c221eac3","__ext.io_kubernetes_workload_name":"bkm-daemonset-worker","__ext.io_kubernetes_workload_type":"DaemonSet","__ext.labels.app_kubernetes_io_component":"bkmonitorbeat","__ext.labels.controller_revision_hash":"6b87cb95fc","__ext.labels.pod_template_generation":"14","__highlight":{"log":["2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]"]},"__index":"5000140_bklog_container_log_demo_analysis","__result_table":"5000140_bklog_container_log_demo_analysis.doris","__shard_key__":29042334000,"_endtime_":"2025-03-21 14:54:03","_starttime_":"2025-03-21 14:54:03","_timestamp_":1742540043000,"_value_":267382,"bk_host_id":267382,"cid":"436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e","cloudid":0,"dteventtime":"2025-03-21 14:54:03","dteventtimestamp":1742540043000,"gseindex":7451424,"iterationindex":1,"level":"INFO","localtime":"2025-03-21 14:54:12","log":"2025-03-21 06:54:03.766\tINFO\t[metricbeat] bkm_metricbeat_scrape_duration_seconds{} 0.002395; kvs=[uri=(http://:10251/metrics)]","logtime":"2025-03-21 06:54:03.766","path":"/data/bcs/service/docker/containers/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e/436cf7ea65b29fb0280c89a774056ac8420ee67d2ba485cfb2932e15c15d9e4e-json.log","thedate":20250321,"time":1742540045}]`, }, + "query raw by doris use condition in highlight": { + query: &metadata.Query{ + TableID: "2_bklog.bklog_pure_v4_log_doris_for_unify_query", + DB: "2_bklog_pure_v4_log_doris_for_unify_query", + Measurement: "doris", + MetricName: "dtEventTimeStamp", + Field: "dtEventTimeStamp", + DataLabel: "log_index_set_1183", + AllConditions: metadata.AllConditions{ + { + {DimensionName: "message", Value: []string{"Bk-Query-Source"}, Operator: "contains"}, + }, + { + {DimensionName: "level", Value: []string{"error", "info"}, Operator: "contains"}, + }, + }, + Size: 5, + HighLight: &metadata.HighLight{ + Enable: true, + }, + }, + expected: `[{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":0,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.643Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[204] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-ed105a4a519bddd9-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__2]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:2_bkapm_metric_mandotest.__default__:trpc.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.643Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":2,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.790Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-1b008865dfe232ab-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745234700\",\"end\":\"1745234900\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.790Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":4,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:00.855Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[202] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-6c1e5853f8d7c5ea-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__7]], body: {\"promql\":\"count by (target) ({__name__=~\\\"custom:7_bkapm_metric_bk_itsm.__default__:trpc.*\\\"})\",\"start\":\"1745235100\",\"end\":\"1745235300\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:00.855Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:361","gseIndex":4281730,"iterationIndex":6,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:01.030Z\tinfo\thttp/handler.go:361\t[9a8222f1a3407f97f351207752953cb5] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[backend] Connection:[keep-alive] Content-Length:[220] Content-Type:[application/json] Traceparent:[00-9a8222f1a3407f97f351207752953cb5-d3159692d865fe24-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bksaas__capp512]], body: {\"promql\":\"count by (service_name) ({__name__=~\\\"custom:bkapm_-59_metric_bkapp_capp512_stag_21.__default__:.*\\\"})\",\"start\":\"1745234900\",\"end\":\"1745235100\",\"step\":\"200s\",\"bk_biz_ids\":null,\"look_back_delta\":\"\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:01.030Z","thedate":20250421,"time":"1745234704","trace_id":"9a8222f1a3407f97f351207752953cb5"},{"__data_label":"log_index_set_1183","__highlight":{"level":["info"],"message":[" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}"]},"__index":"2_bklog_pure_v4_log_doris_for_unify_query","__result_table":"2_bklog.bklog_pure_v4_log_doris_for_unify_query","__shard_key__":29087245464,"_timestamp_":1745234704000,"_value_":1745234704000,"cloudId":0,"dtEventTime":"2025-04-21 19:25:04","dtEventTimeStamp":1745234704000,"file":"http/handler.go:305","gseIndex":4281730,"iterationIndex":8,"level":"info","localTime":"2025-04-21 19:25:04","log":"2025-04-21T11:25:01.199Z\tinfo\thttp/handler.go:305\t[adb84ecc380008245cdb800b6fd54d7f] header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","message":" header: map[Accept:[*/*] Accept-Encoding:[gzip, deflate] Bk-Query-Source:[username:admin] Connection:[keep-alive] Content-Length:[686] Content-Type:[application/json] Traceparent:[00-adb84ecc380008245cdb800b6fd54d7f-ddc4638680c14719-00] User-Agent:[python-requests/2.31.0] X-Bk-Scope-Space-Uid:[bkcc__11]], body: {\"space_uid\":\"bkcc__11\",\"query_list\":[{\"table_id\":\"system.cpu_summary\",\"field_name\":\"usage\",\"is_regexp\":false,\"function\":[{\"method\":\"mean\",\"without\":false,\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"]}],\"time_aggregation\":{\"function\":\"avg_over_time\",\"window\":\"60s\"},\"is_dom_sampled\":false,\"reference_name\":\"a\",\"dimensions\":[\"bk_target_ip\",\"bk_target_cloud_id\"],\"conditions\":{},\"keep_columns\":[\"_time\",\"a\",\"bk_target_ip\",\"bk_target_cloud_id\"],\"query_string\":\"\"}],\"metric_merge\":\"a\",\"start_time\":\"1745234520\",\"end_time\":\"1745234700\",\"step\":\"60s\",\"timezone\":\"Asia/Shanghai\",\"instant\":false}","path":"/var/host/data/bcs/lib/docker/containers/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93/5d0cff8ae531973edec39aa9989439c6371357491c3c7791d920e6f5569d5c93-json.log","report_time":"2025-04-21T11:25:01.199Z","thedate":20250421,"time":"1745234704","trace_id":"adb84ecc380008245cdb800b6fd54d7f"}]`, + }, } { t.Run(name, func(t *testing.T) { ctx = metadata.InitHashID(ctx) From 241a302545f0df2db44e7f25f1ced1363b65e669 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 25 Apr 2025 17:20:28 +0800 Subject: [PATCH 100/104] =?UTF-8?q?feat:=20=E6=97=A5=E5=BF=97=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81doris=20#1010158081121933504?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/bksql/format.go | 2 +- pkg/unify-query/tsdb/bksql/instance_test.go | 2 +- pkg/unify-query/tsdb/bksql/sqlExpr/base.go | 40 +++---------------- .../tsdb/bksql/sqlExpr/base_test.go | 2 +- pkg/unify-query/tsdb/bksql/sqlExpr/doris.go | 4 -- .../tsdb/bksql/sqlExpr/doris_test.go | 4 +- 6 files changed, 11 insertions(+), 43 deletions(-) diff --git a/pkg/unify-query/tsdb/bksql/format.go b/pkg/unify-query/tsdb/bksql/format.go index 42c8694a5..45fbbbf66 100644 --- a/pkg/unify-query/tsdb/bksql/format.go +++ b/pkg/unify-query/tsdb/bksql/format.go @@ -89,7 +89,7 @@ func NewQueryFactory(ctx context.Context, query *metadata.Query) *QueryFactory { f.timeField = dtEventTimeStamp } - f.expr = sqlExpr.GetSQLExpr(f.query.Measurement). + f.expr = sqlExpr.NewSQLExpr(f.query.Measurement). WithInternalFields(f.timeField, query.Field). WithEncode(metadata.GetPromDataFormat(ctx).EncodeFunc()) diff --git a/pkg/unify-query/tsdb/bksql/instance_test.go b/pkg/unify-query/tsdb/bksql/instance_test.go index 5de17e8b1..46d9b4f94 100644 --- a/pkg/unify-query/tsdb/bksql/instance_test.go +++ b/pkg/unify-query/tsdb/bksql/instance_test.go @@ -836,7 +836,7 @@ func TestInstance_bkSql(t *testing.T) { "text": sqlExpr.DorisTypeText, } - condition, err := sqlExpr.GetSQLExpr(c.query.Measurement).WithFieldsMap(fieldsMap).ParserAllConditions(c.query.AllConditions) + condition, err := sqlExpr.NewSQLExpr(c.query.Measurement).WithFieldsMap(fieldsMap).ParserAllConditions(c.query.AllConditions) assert.Nil(t, err) if err == nil { assert.Equal(t, c.query.BkSqlCondition, condition) diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go index cfb2d9001..a271b82ee 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base.go @@ -12,7 +12,6 @@ package sqlExpr import ( "fmt" "strings" - "sync" "time" "github.com/TencentBlueKing/bkmonitor-datalink/pkg/unify-query/metadata" @@ -71,12 +70,9 @@ type SQLExpr interface { // SQL表达式注册管理相关变量 var ( _ SQLExpr = (*DefaultSQLExpr)(nil) // 接口实现检查 - - lock sync.RWMutex // 读写锁用于并发安全 - exprMap = make(map[string]SQLExpr) // 存储注册的SQL表达式实现 ) -// GetSQLExpr 获取指定key的SQL表达式实现 +// NewSQLExpr 获取指定key的SQL表达式实现 // 参数： // // key - 注册时使用的标识符 @@ -84,39 +80,15 @@ var ( // 返回值： // // 找到返回对应实现，未找到返回默认实现 -func GetSQLExpr(key string) SQLExpr { - lock.RLock() - defer lock.RUnlock() - if sqlExpr, ok := exprMap[key]; ok { - return sqlExpr - } else { +func NewSQLExpr(key string) SQLExpr { + switch key { + case Doris: + return &DorisSQLExpr{} + default: return &DefaultSQLExpr{} } } -// Register 注册SQL表达式实现 -// 参数： -// -// key - 实现标识符 -// sqlExpr - 具体的SQL表达式实现 -func Register(key string, sqlExpr SQLExpr) { - lock.Lock() - defer lock.Unlock() - exprMap[key] = sqlExpr -} - -// UnRegister 注销指定key的SQL表达式实现 -// 参数： -// -// key - 要注销的实现标识符 -func UnRegister(key string) { - lock.Lock() - defer lock.Unlock() - if _, ok := exprMap[key]; ok { - delete(exprMap, key) - } -} - // DefaultSQLExpr SQL表达式默认实现 type DefaultSQLExpr struct { encodeFunc func(string) string diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go index 7736190e5..fdd201770 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/base_test.go @@ -20,7 +20,7 @@ import ( // TestParserAllConditions 测试全条件解析的主测试函数 func TestParserAllConditions(t *testing.T) { - d := sqlExpr.GetSQLExpr("") + d := sqlExpr.NewSQLExpr("") t.Run("空条件测试", func(t *testing.T) { conditions := metadata.AllConditions{} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go index 7ac02e43e..13ad0ced1 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris.go @@ -550,7 +550,3 @@ func (d *DorisSQLExpr) valueTransform(s string) string { } return s } - -func init() { - Register(Doris, &DorisSQLExpr{}) -} diff --git a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go index ba70a5df6..c3d3fb9e9 100644 --- a/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go +++ b/pkg/unify-query/tsdb/bksql/sqlExpr/doris_test.go @@ -101,7 +101,7 @@ func TestDorisSQLExpr_ParserQueryString(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + got, err := sqlExpr.NewSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ "text": sqlExpr.DorisTypeText, }).ParserQueryString(tt.input) if err != nil { @@ -242,7 +242,7 @@ func TestDorisSQLExpr_ParserAllConditions(t *testing.T) { }, } - e := sqlExpr.GetSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ + e := sqlExpr.NewSQLExpr(sqlExpr.Doris).WithFieldsMap(map[string]string{ "object.field": sqlExpr.DorisTypeText, }) From c2100e7ee736477e206affc5135b78cbf7acd65d Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 30 Apr 2025 18:16:27 +0800 Subject: [PATCH 101/104] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20es=20?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=20qr=20=E4=B8=BA=E7=A9=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#1010158081141941944?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/format.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index 5ec296c1d..f41ceb0e9 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -399,7 +399,9 @@ func (f *FormatFactory) nestedAgg(key string) { // AggDataFormat 解析 es 的聚合计算 func (f *FormatFactory) AggDataFormat(data elastic.Aggregations, metricLabel *prompb.Label) (*prompb.QueryResult, error) { if data == nil { - return nil, nil + return &prompb.QueryResult{ + Timeseries: []*prompb.TimeSeries{}, + }, nil } defer func() { From 5e7f5f1e9647623727ffa3cf4609db15e138f880 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Wed, 30 Apr 2025 18:24:46 +0800 Subject: [PATCH 102/104] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20es=20?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=20qr=20=E4=B8=BA=E7=A9=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#1010158081141941944?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/instance.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/unify-query/tsdb/elasticsearch/instance.go b/pkg/unify-query/tsdb/elasticsearch/instance.go index c4f7911da..dc40ba5ce 100644 --- a/pkg/unify-query/tsdb/elasticsearch/instance.go +++ b/pkg/unify-query/tsdb/elasticsearch/instance.go @@ -394,6 +394,10 @@ func (i *Instance) queryWithAgg(ctx context.Context, qo *queryOption, fact *Form return storage.ErrSeriesSet(err) } + if sr == nil || sr.Aggregations == nil { + return storage.EmptySeriesSet() + } + // 如果是非时间聚合计算，则无需进行指标名的拼接作用 qr, err := fact.AggDataFormat(sr.Aggregations, metricLabel) if err != nil { From 478281e91159586f91ddb69691d406d1f3fb8cc3 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Thu, 8 May 2025 21:59:58 +0800 Subject: [PATCH 103/104] =?UTF-8?q?feat:=20es=20=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=89=8D=E7=BC=80=E6=A8=A1=E5=BC=8F=20#10101?= =?UTF-8?q?58081123906702?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/metadata/struct.go | 6 +++++ .../query/structured/condition_field.go | 4 ++++ pkg/unify-query/query/structured/query_ts.go | 4 ++++ pkg/unify-query/tsdb/elasticsearch/format.go | 12 ++++++++-- .../tsdb/elasticsearch/format_test.go | 24 +++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pkg/unify-query/metadata/struct.go b/pkg/unify-query/metadata/struct.go index 45b85e1cc..ce9aa44fc 100644 --- a/pkg/unify-query/metadata/struct.go +++ b/pkg/unify-query/metadata/struct.go @@ -166,6 +166,12 @@ type ConditionField struct { // IsWildcard 是否是通配符 IsWildcard bool + + // IsPrefix 是否是前缀匹配 + IsPrefix bool + + // IsSuffix 是否是后缀匹配 + IsSuffix bool } // TimeAggregation 时间聚合字段 diff --git a/pkg/unify-query/query/structured/condition_field.go b/pkg/unify-query/query/structured/condition_field.go index 0214d1906..1642154be 100644 --- a/pkg/unify-query/query/structured/condition_field.go +++ b/pkg/unify-query/query/structured/condition_field.go @@ -84,6 +84,10 @@ type ConditionField struct { Operator string `json:"op" example:"contains"` // IsWildcard 是否是通配符 IsWildcard bool `json:"is_wildcard,omitempty"` + // IsPrefix 是否是前缀 + IsPrefix bool `json:"is_prefix,omitempty"` + // IsSuffix 是否是后缀 + IsSuffix bool `json:"is_suffix,omitempty"` } // String diff --git a/pkg/unify-query/query/structured/query_ts.go b/pkg/unify-query/query/structured/query_ts.go index 7a229de92..a1cb63a90 100644 --- a/pkg/unify-query/query/structured/query_ts.go +++ b/pkg/unify-query/query/structured/query_ts.go @@ -201,6 +201,8 @@ func (q *QueryTs) ToQueryClusterMetric(ctx context.Context) (*metadata.QueryClus Value: cond.Value, Operator: cond.Operator, IsWildcard: cond.IsWildcard, + IsPrefix: cond.IsPrefix, + IsSuffix: cond.IsSuffix, }) } queryConditions = append(queryConditions, queryConds) @@ -931,6 +933,8 @@ func (q *Query) BuildMetadataQuery( Value: c.Value, Operator: c.Operator, IsWildcard: c.IsWildcard, + IsPrefix: c.IsPrefix, + IsSuffix: c.IsSuffix, } } query.AllConditions[i] = conds diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index f41ceb0e9..09cb6d60f 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -788,14 +788,22 @@ func (f *FormatFactory) Query(allConditions metadata.AllConditions) (elastic.Que // 非空才进行验证 switch con.Operator { case structured.ConditionEqual, structured.ConditionNotEqual: - query = elastic.NewMatchPhraseQuery(key, value) + if con.IsPrefix { + query = elastic.NewMatchPhrasePrefixQuery(key, value) + } else { + query = elastic.NewMatchPhraseQuery(key, value) + } case structured.ConditionContains, structured.ConditionNotContains: if fieldType == KeyWord { value = fmt.Sprintf("*%s*", value) } if !con.IsWildcard && fieldType == Text { - query = elastic.NewMatchPhraseQuery(key, value) + if con.IsPrefix { + query = elastic.NewMatchPhrasePrefixQuery(key, value) + } else { + query = elastic.NewMatchPhraseQuery(key, value) + } } else { query = elastic.NewWildcardQuery(key, value) } diff --git a/pkg/unify-query/tsdb/elasticsearch/format_test.go b/pkg/unify-query/tsdb/elasticsearch/format_test.go index 5d19b94c6..f320b2f8c 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/format_test.go @@ -115,6 +115,30 @@ func TestFormatFactory_Query(t *testing.T) { }, expected: `{"query":{"bool":{"must":[{"match_phrase":{"key-1":{"query":"val-1"}}},{"match_phrase":{"key-2":{"query":"val-2"}}},{"match_phrase":{"key-3":{"query":"val-3"}}}]}}}`, }, + "query with prefix and suffix": { + conditions: metadata.AllConditions{ + { + { + DimensionName: "key-1", + Value: []string{"val-1"}, + Operator: structured.ConditionEqual, + IsPrefix: true, + }, + { + DimensionName: "key-2", + Value: []string{"val-2"}, + Operator: structured.ConditionEqual, + IsSuffix: true, + }, + { + DimensionName: "key-3", + Value: []string{"val-3"}, + Operator: structured.ConditionEqual, + }, + }, + }, + expected: `{"query":{"bool":{"must":[{"match_phrase_prefix":{"key-1":{"query":"val-1"}}},{"match_phrase":{"key-2":{"query":"val-2"}}},{"match_phrase":{"key-3":{"query":"val-3"}}}]}}}`, + }, "nested query": { conditions: metadata.AllConditions{ { From 0ccce9e0963a19905f648a88674fb31598d520a9 Mon Sep 17 00:00:00 2001 From: shamcleren Date: Fri, 9 May 2025 11:41:21 +0800 Subject: [PATCH 104/104] =?UTF-8?q?feat:=20es=20=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E8=81=9A=E5=90=88=E6=97=B6=E9=97=B4=E9=80=82=E9=85=8D=E9=9D=9E?= =?UTF-8?q?=E6=AF=AB=E7=A7=92=E5=8D=95=E4=BD=8D=20#1010158081123913773?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/unify-query/tsdb/elasticsearch/format.go | 41 ++++++++-- .../tsdb/elasticsearch/format_test.go | 74 ++++++++++++++++++- .../tsdb/elasticsearch/functions.go | 20 ++--- .../tsdb/elasticsearch/functions_test.go | 4 +- 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/pkg/unify-query/tsdb/elasticsearch/format.go b/pkg/unify-query/tsdb/elasticsearch/format.go index f41ceb0e9..3ab2c6304 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format.go +++ b/pkg/unify-query/tsdb/elasticsearch/format.go @@ -148,7 +148,7 @@ type ValueAgg struct { type TimeAgg struct { Name string - Window string + Window time.Duration Timezone string } @@ -211,6 +211,22 @@ func (f *FormatFactory) WithIsReference(isReference bool) *FormatFactory { return f } +func (f *FormatFactory) toFixInterval(window time.Duration) (string, error) { + switch f.timeField.Unit { + case function.Second: + window /= 1e3 + case function.Microsecond: + window *= 1e3 + case function.Nanosecond: + window *= 1e6 + } + + if window.Milliseconds() < 1 { + return "", fmt.Errorf("date histogram aggregation interval must be greater than 0ms") + } + return shortDur(window), nil +} + func (f *FormatFactory) toMillisecond(i int64) int64 { switch f.timeField.Unit { case function.Second: @@ -335,7 +351,7 @@ func (f *FormatFactory) RangeQuery() (elastic.Query, error) { return query, err } -func (f *FormatFactory) timeAgg(name string, window, timezone string) { +func (f *FormatFactory) timeAgg(name string, window time.Duration, timezone string) { f.aggInfoList = append( f.aggInfoList, TimeAgg{ Name: name, Window: window, Timezone: timezone, @@ -583,9 +599,20 @@ func (f *FormatFactory) Agg() (name string, agg elastic.Aggregation, err error) case TimeAgg: curName := info.Name + var interval string + + if f.timeField.Type == TimeFieldTypeInt { + interval, err = f.toFixInterval(info.Window) + if err != nil { + return + } + } else { + interval = shortDur(info.Window) + } + curAgg := elastic.NewDateHistogramAggregation(). - Field(f.timeField.Name).Interval(info.Window).MinDocCount(0). - ExtendedBounds(f.TimeFieldUnix(f.start), f.TimeFieldUnix(f.end)) + Field(f.timeField.Name).Interval(interval).MinDocCount(0). + ExtendedBounds(f.timeFieldUnix(f.start), f.timeFieldUnix(f.end)) // https://github.com/elastic/elasticsearch/issues/42270 非date类型不支持timezone, time format也无效 if f.timeField.Type == TimeFieldTypeTime { curAgg = curAgg.TimeZone(info.Timezone) @@ -654,14 +681,14 @@ func (f *FormatFactory) EsAgg(aggregates metadata.Aggregates) (string, elastic.A for _, am := range aggregates { switch am.Name { case DateHistogram: - f.timeAgg(f.timeField.Name, shortDur(am.Window), am.TimeZone) + f.timeAgg(f.timeField.Name, am.Window, am.TimeZone) case Max, Min, Avg, Sum, Count, Cardinality, Percentiles: f.valueAgg(FieldValue, am.Name, am.Args...) f.nestedAgg(f.valueField) if am.Window > 0 && !am.Without { // 增加时间函数 - f.timeAgg(f.timeField.Name, shortDur(am.Window), am.TimeZone) + f.timeAgg(f.timeField.Name, am.Window, am.TimeZone) } for idx, dim := range am.Dimensions { @@ -700,7 +727,7 @@ func (f *FormatFactory) Orders() metadata.Orders { return orders } -func (f *FormatFactory) TimeFieldUnix(t time.Time) (u int64) { +func (f *FormatFactory) timeFieldUnix(t time.Time) (u int64) { switch f.timeField.Unit { case function.Millisecond: u = t.UnixMilli() diff --git a/pkg/unify-query/tsdb/elasticsearch/format_test.go b/pkg/unify-query/tsdb/elasticsearch/format_test.go index 5d19b94c6..c48fdf6f8 100644 --- a/pkg/unify-query/tsdb/elasticsearch/format_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/format_test.go @@ -343,7 +343,7 @@ func TestFormatFactory_RangeQueryAndAggregates(t *testing.T) { TimeZone: "Asia/ShangHai", }, }, - expected: `{"aggregations":{"gseIndex":{"aggregations":{"time":{"aggregations":{"_value":{"value_count":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":1721046420,"min":1721024820},"field":"time","interval":"1m","min_doc_count":0}}},"terms":{"field":"gseIndex","missing":" "}}},"query":{"range":{"time":{"from":1721024820,"include_lower":true,"include_upper":true,"to":1721046420}}}}`, + expected: `{"aggregations":{"gseIndex":{"aggregations":{"time":{"aggregations":{"_value":{"value_count":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":1721046420,"min":1721024820},"field":"time","interval":"60ms","min_doc_count":0}}},"terms":{"field":"gseIndex","missing":" "}}},"query":{"range":{"time":{"from":1721024820,"include_lower":true,"include_upper":true,"to":1721046420}}}}`, }, "aggregate millisecond int field": { timeField: metadata.TimeField{ @@ -451,3 +451,75 @@ func TestFormatFactory_AggDataFormat(t *testing.T) { }) } } + +func TestToFixInterval(t *testing.T) { + tests := []struct { + name string + timeUnit string + window time.Duration + want string + wantError bool + }{ + { + name: "second unit should error", + timeUnit: function.Second, + window: time.Second, + want: "1ms", + }, + { + name: "window less than 1 should error", + timeUnit: function.Millisecond, + window: time.Microsecond, // 0.001ms + wantError: true, + }, + { + name: "microsecond unit conversion", + timeUnit: function.Microsecond, + window: time.Millisecond, // 1ms = 1000us + want: "1s", + }, + { + name: "nanosecond unit conversion", + timeUnit: function.Nanosecond, + window: time.Millisecond, // 1ms = 1000000ns + want: "1000s", + }, + { + name: "microsecond unit no conversion", + timeUnit: function.Microsecond, + window: time.Minute, + want: "1000m", + }, + { + name: "microsecond unit no conversion", + timeUnit: function.Microsecond, + window: time.Hour * 6, + want: "250d", // 250d = 6000h + }, + { + name: "nanosecond unit no conversion", + timeUnit: function.Nanosecond, + window: time.Minute, + want: "1000000m", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &FormatFactory{ + timeField: metadata.TimeField{ + Unit: tt.timeUnit, + }, + } + + got, err := f.toFixInterval(tt.window) + if (err != nil) != tt.wantError { + t.Errorf("toFixInterval() error = %v, wantError %v", err, tt.wantError) + return + } + if !tt.wantError && got != tt.want { + t.Errorf("toFixInterval() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/unify-query/tsdb/elasticsearch/functions.go b/pkg/unify-query/tsdb/elasticsearch/functions.go index 016afe183..415b23583 100644 --- a/pkg/unify-query/tsdb/elasticsearch/functions.go +++ b/pkg/unify-query/tsdb/elasticsearch/functions.go @@ -65,19 +65,21 @@ func parseSizeString(sizeStr string) (int64, error) { } func shortDur(d time.Duration) string { - nd := int(d.Seconds()) + nd := d.Milliseconds() if nd == 0 { - return "0s" + return "0ms" } - if nd%(24*60*60) == 0 { - return fmt.Sprintf("%dd", nd/24/60/60) - } else if nd%(60*60) == 0 { - return fmt.Sprintf("%dh", nd/60/60) - } else if nd%60 == 0 { - return fmt.Sprintf("%dm", nd/60) + if nd%(time.Hour.Milliseconds()*24) == 0 { + return fmt.Sprintf("%dd", nd/time.Hour.Milliseconds()/24) + } else if nd%(time.Hour.Milliseconds()) == 0 { + return fmt.Sprintf("%dh", nd/time.Hour.Milliseconds()) + } else if nd%(time.Minute.Milliseconds()) == 0 { + return fmt.Sprintf("%dm", nd/time.Minute.Milliseconds()) + } else if nd%(time.Second.Milliseconds()) == 0 { + return fmt.Sprintf("%ds", nd/time.Second.Milliseconds()) } else { - return fmt.Sprintf("%ds", nd) + return fmt.Sprintf("%dms", nd) } } diff --git a/pkg/unify-query/tsdb/elasticsearch/functions_test.go b/pkg/unify-query/tsdb/elasticsearch/functions_test.go index d3d528b80..cdb6194ab 100644 --- a/pkg/unify-query/tsdb/elasticsearch/functions_test.go +++ b/pkg/unify-query/tsdb/elasticsearch/functions_test.go @@ -70,7 +70,7 @@ func TestShortDur(t *testing.T) { { name: "0 seconds", duration: 0 * time.Second, - expected: "0s", + expected: "0ms", }, { name: "1 day", @@ -87,7 +87,7 @@ func TestShortDur(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := shortDur(tt.duration) - assert.Equal(t, got, tt.expected) + assert.Equal(t, tt.expected, got) }) } }