Skip to content

Commit 5875d7d

Browse files
srikanthccvsrikanthccv
authored
Merge branch 'main' into fix-7233
2 parents 00f9ba1 + 8f2e8cc commit 5875d7d

File tree

5 files changed

+221
-3
lines changed

5 files changed

+221
-3
lines changed

‎pkg/query-service/app/clickhouseReader/options.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ const (
4848
defaultTraceLocalTableName string = "signoz_index_v3"
4949
defaultTraceResourceTableV3 string = "distributed_traces_v3_resource"
5050
defaultTraceSummaryTable string = "distributed_trace_summary"
51+
52+
defaultMetadataDB string = "signoz_metadata"
53+
defaultMetadataTable string = "distributed_attributes_metadata"
5154
)
5255

5356
// NamespaceConfig is Clickhouse's internal configuration data
@@ -88,6 +91,8 @@ type namespaceConfig struct {
8891
TraceLocalTableNameV3 string
8992
TraceResourceTableV3 string
9093
TraceSummaryTable string
94+
MetadataDB string
95+
MetadataTable string
9196
}
9297

9398
// Connecto defines how to connect to the database
@@ -141,6 +146,8 @@ func NewOptions(
141146
TraceLocalTableNameV3: defaultTraceLocalTableName,
142147
TraceResourceTableV3: defaultTraceResourceTableV3,
143148
TraceSummaryTable: defaultTraceSummaryTable,
149+
MetadataDB: defaultMetadataDB,
150+
MetadataTable: defaultMetadataTable,
144151
},
145152
others: make(map[string]*namespaceConfig, len(otherNamespaces)),
146153
}

‎pkg/query-service/app/clickhouseReader/reader.go

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@ type ClickHouseReader struct {
164164

165165
fluxIntervalForTraceDetail time.Duration
166166
cache cache.Cache
167+
metadataDB string
168+
metadataTable string
167169
}
168170

169171
// NewTraceReader returns a TraceReader for the database
@@ -259,6 +261,8 @@ func NewReaderFromClickhouseConnection(
259261

260262
fluxIntervalForTraceDetail: fluxIntervalForTraceDetail,
261263
cache: cache,
264+
metadataDB: options.primary.MetadataDB,
265+
metadataTable: options.primary.MetadataTable,
262266
}
263267
}
264268

@@ -4126,6 +4130,97 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt
41264130
return &response, nil
41274131
}
41284132

4133+
func (r *ClickHouseReader) FetchRelatedValues(ctx context.Context, req *v3.FilterAttributeValueRequest) ([]string, error) {
4134+
var andConditions []string
4135+
4136+
andConditions = append(andConditions, fmt.Sprintf("unix_milli >= %d", req.StartTimeMillis))
4137+
andConditions = append(andConditions, fmt.Sprintf("unix_milli <= %d", req.EndTimeMillis))
4138+
4139+
if len(req.ExistingFilterItems) != 0 {
4140+
for _, item := range req.ExistingFilterItems {
4141+
// we only support string for related values
4142+
if item.Key.DataType != v3.AttributeKeyDataTypeString {
4143+
continue
4144+
}
4145+
4146+
var colName string
4147+
switch item.Key.Type {
4148+
case v3.AttributeKeyTypeResource:
4149+
colName = "resource_attributes"
4150+
case v3.AttributeKeyTypeTag:
4151+
colName = "attributes"
4152+
default:
4153+
// we only support resource and tag for related values as of now
4154+
continue
4155+
}
4156+
// IN doesn't make use of map value index, we convert it to = or !=
4157+
operator := item.Operator
4158+
if v3.FilterOperator(strings.ToLower(string(item.Operator))) == v3.FilterOperatorIn {
4159+
operator = "="
4160+
} else if v3.FilterOperator(strings.ToLower(string(item.Operator))) == v3.FilterOperatorNotIn {
4161+
operator = "!="
4162+
}
4163+
addCondition := func(val string) {
4164+
andConditions = append(andConditions, fmt.Sprintf("mapContains(%s, '%s') AND %s['%s'] %s %s", colName, item.Key.Key, colName, item.Key.Key, operator, val))
4165+
}
4166+
switch v := item.Value.(type) {
4167+
case string:
4168+
fmtVal := utils.ClickHouseFormattedValue(v)
4169+
addCondition(fmtVal)
4170+
case []string:
4171+
for _, val := range v {
4172+
fmtVal := utils.ClickHouseFormattedValue(val)
4173+
addCondition(fmtVal)
4174+
}
4175+
case []interface{}:
4176+
for _, val := range v {
4177+
fmtVal := utils.ClickHouseFormattedValue(val)
4178+
addCondition(fmtVal)
4179+
}
4180+
}
4181+
}
4182+
}
4183+
whereClause := strings.Join(andConditions, " AND ")
4184+
4185+
var selectColumn string
4186+
switch req.TagType {
4187+
case v3.TagTypeResource:
4188+
selectColumn = "resource_attributes" + "['" + req.FilterAttributeKey + "']"
4189+
case v3.TagTypeTag:
4190+
selectColumn = "attributes" + "['" + req.FilterAttributeKey + "']"
4191+
default:
4192+
selectColumn = "attributes" + "['" + req.FilterAttributeKey + "']"
4193+
}
4194+
4195+
filterSubQuery := fmt.Sprintf(
4196+
"SELECT DISTINCT %s FROM %s.%s WHERE %s LIMIT 100",
4197+
selectColumn,
4198+
r.metadataDB,
4199+
r.metadataTable,
4200+
whereClause,
4201+
)
4202+
zap.L().Debug("filterSubQuery for related values", zap.String("query", filterSubQuery))
4203+
4204+
rows, err := r.db.Query(ctx, filterSubQuery)
4205+
if err != nil {
4206+
return nil, fmt.Errorf("error while executing query: %s", err.Error())
4207+
}
4208+
defer rows.Close()
4209+
4210+
var attributeValues []string
4211+
for rows.Next() {
4212+
var value string
4213+
if err := rows.Scan(&value); err != nil {
4214+
return nil, fmt.Errorf("error while scanning rows: %s", err.Error())
4215+
}
4216+
if value != "" {
4217+
attributeValues = append(attributeValues, value)
4218+
}
4219+
}
4220+
4221+
return attributeValues, nil
4222+
}
4223+
41294224
func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) {
41304225
var err error
41314226
var filterValueColumn string
@@ -4227,6 +4322,13 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi
42274322
}
42284323
}
42294324

4325+
if req.IncludeRelated {
4326+
relatedValues, _ := r.FetchRelatedValues(ctx, req)
4327+
attributeValues.RelatedValues = &v3.FilterAttributeValueResponse{
4328+
StringAttributeValues: relatedValues,
4329+
}
4330+
}
4331+
42304332
return &attributeValues, nil
42314333

42324334
}
@@ -4907,6 +5009,13 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.
49075009
}
49085010
}
49095011

5012+
if req.IncludeRelated {
5013+
relatedValues, _ := r.FetchRelatedValues(ctx, req)
5014+
attributeValues.RelatedValues = &v3.FilterAttributeValueResponse{
5015+
StringAttributeValues: relatedValues,
5016+
}
5017+
}
5018+
49105019
return &attributeValues, nil
49115020
}
49125021

‎pkg/query-service/app/http_handler.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,12 @@ func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMid
395395
withCacheControl(AutoCompleteCacheControlAge, aH.autoCompleteAttributeKeys))).Methods(http.MethodGet)
396396
subRouter.HandleFunc("/autocomplete/attribute_values", am.ViewAccess(
397397
withCacheControl(AutoCompleteCacheControlAge, aH.autoCompleteAttributeValues))).Methods(http.MethodGet)
398+
399+
// autocomplete with filters using new endpoints
400+
// Note: eventually all autocomplete APIs should be migrated to new endpoint with appropriate filters, deprecating the older ones
401+
402+
subRouter.HandleFunc("/auto_complete/attribute_values", am.ViewAccess(aH.autoCompleteAttributeValuesPost)).Methods(http.MethodPost)
403+
398404
subRouter.HandleFunc("/query_range", am.ViewAccess(aH.QueryRangeV3)).Methods(http.MethodPost)
399405
subRouter.HandleFunc("/query_range/format", am.ViewAccess(aH.QueryRangeV3Format)).Methods(http.MethodPost)
400406

@@ -4834,6 +4840,35 @@ func (aH *APIHandler) autoCompleteAttributeValues(w http.ResponseWriter, r *http
48344840
aH.Respond(w, response)
48354841
}
48364842

4843+
func (aH *APIHandler) autoCompleteAttributeValuesPost(w http.ResponseWriter, r *http.Request) {
4844+
var response *v3.FilterAttributeValueResponse
4845+
req, err := parseFilterAttributeValueRequestBody(r)
4846+
4847+
if err != nil {
4848+
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
4849+
return
4850+
}
4851+
4852+
switch req.DataSource {
4853+
case v3.DataSourceMetrics:
4854+
response, err = aH.reader.GetMetricAttributeValues(r.Context(), req)
4855+
case v3.DataSourceLogs:
4856+
response, err = aH.reader.GetLogAttributeValues(r.Context(), req)
4857+
case v3.DataSourceTraces:
4858+
response, err = aH.reader.GetTraceAttributeValues(r.Context(), req)
4859+
default:
4860+
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid data source")}, nil)
4861+
return
4862+
}
4863+
4864+
if err != nil {
4865+
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
4866+
return
4867+
}
4868+
4869+
aH.Respond(w, response)
4870+
}
4871+
48374872
func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3) (map[string]v3.AttributeKey, error) {
48384873
data := map[string]v3.AttributeKey{}
48394874
for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {

‎pkg/query-service/app/parser.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,25 @@ func parseFilterAttributeKeyRequest(r *http.Request) (*v3.FilterAttributeKeyRequ
741741
return &req, nil
742742
}
743743

744+
func parseFilterAttributeValueRequestBody(r *http.Request) (*v3.FilterAttributeValueRequest, error) {
745+
746+
var req v3.FilterAttributeValueRequest
747+
748+
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
749+
return nil, err
750+
}
751+
752+
if err := req.Validate(); err != nil {
753+
return nil, err
754+
}
755+
756+
// offset by two windows periods for start for better results
757+
req.StartTimeMillis = req.StartTimeMillis - time.Hour.Milliseconds()*6*2
758+
req.EndTimeMillis = req.EndTimeMillis + time.Hour.Milliseconds()*6
759+
760+
return &req, nil
761+
}
762+
744763
func parseFilterAttributeValueRequest(r *http.Request) (*v3.FilterAttributeValueRequest, error) {
745764

746765
var req v3.FilterAttributeValueRequest

‎pkg/query-service/model/v3/v3.go

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,8 @@ func (q AttributeKeyDataType) String() string {
297297
// for a selected aggregate operator, aggregate attribute, filter attribute key
298298
// and search text.
299299
type FilterAttributeValueRequest struct {
300+
StartTimeMillis int64 `json:"startTimeMillis"`
301+
EndTimeMillis int64 `json:"endTimeMillis"`
300302
DataSource DataSource `json:"dataSource"`
301303
AggregateOperator AggregateOperator `json:"aggregateOperator"`
302304
AggregateAttribute string `json:"aggregateAttribute"`
@@ -305,6 +307,51 @@ type FilterAttributeValueRequest struct {
305307
TagType TagType `json:"tagType"`
306308
SearchText string `json:"searchText"`
307309
Limit int `json:"limit"`
310+
ExistingFilterItems []FilterItem `json:"existingFilterItems"`
311+
MetricNames []string `json:"metricNames"`
312+
IncludeRelated bool `json:"includeRelated"`
313+
}
314+
315+
func (f *FilterAttributeValueRequest) Validate() error {
316+
if f.FilterAttributeKey == "" {
317+
return fmt.Errorf("filterAttributeKey is required")
318+
}
319+
320+
if f.StartTimeMillis == 0 {
321+
return fmt.Errorf("startTimeMillis is required")
322+
}
323+
324+
if f.EndTimeMillis == 0 {
325+
return fmt.Errorf("endTimeMillis is required")
326+
}
327+
328+
if f.Limit == 0 {
329+
f.Limit = 100
330+
}
331+
332+
if f.Limit > 1000 {
333+
return fmt.Errorf("limit must be less than 1000")
334+
}
335+
336+
if f.ExistingFilterItems != nil {
337+
for _, value := range f.ExistingFilterItems {
338+
if value.Key.Key == "" {
339+
return fmt.Errorf("existingFilterItems must contain a valid key")
340+
}
341+
}
342+
}
343+
344+
if err := f.DataSource.Validate(); err != nil {
345+
return fmt.Errorf("invalid data source: %w", err)
346+
}
347+
348+
if f.DataSource != DataSourceMetrics {
349+
if err := f.AggregateOperator.Validate(); err != nil {
350+
return fmt.Errorf("invalid aggregate operator: %w", err)
351+
}
352+
}
353+
354+
return nil
308355
}
309356

310357
type AggregateAttributeResponse struct {
@@ -366,9 +413,10 @@ func (a AttributeKey) Validate() error {
366413
}
367414

368415
type FilterAttributeValueResponse struct {
369-
StringAttributeValues []string `json:"stringAttributeValues"`
370-
NumberAttributeValues []interface{} `json:"numberAttributeValues"`
371-
BoolAttributeValues []bool `json:"boolAttributeValues"`
416+
StringAttributeValues []string `json:"stringAttributeValues"`
417+
NumberAttributeValues []interface{} `json:"numberAttributeValues"`
418+
BoolAttributeValues []bool `json:"boolAttributeValues"`
419+
RelatedValues *FilterAttributeValueResponse `json:"relatedValues,omitempty"`
372420
}
373421

374422
type QueryRangeParamsV3 struct {

0 commit comments

Comments
 (0)